Buttons still visible after Destroy() but say "deleted gameobject" in hierarchy

Using Unity 2018.3.11f. I am trying to show a pickup button when the player walks over an item, or several, and then removes the button if the player walks to far away. This works great, except the actual button only works sometimes. In Game View, the UI Buttons are added and removed at the correct distances. When going to click on the Buttons, however, sometimes the Buttons are clickable and work great, other times the Buttons don't even show up as highlighted when hovered over. When checking the scene view, the UI Button GameObjects are correctly located as children of the Canvas, but when clicked in the Hierarchy, nothing shows up. I managed to see it say "deleted gameobject" in place of the Button name when I clicked one of the buttons once, which flashed back to the original name. This happens both in editor Play Mode and in a standalone build. Here is the code I am using for creating and destroying the Buttons:

 // Update is called once per frame
     void Update()
     {
         for(int i=nearbyInteractables.Count-1;i>=0;i--)
         {
             if (nearbyInteractables[i].interactable != null)
             {
                 if (Vector2.Distance(nearbyInteractables[i].interactable.transform.position,transform.position) > interactionRange)
                 {
                     Destroy(nearbyInteractables[i].itemInteractionButton.gameObject);
                     nearbyInteractables.RemoveAt(i);
                 }
             }
             else
             {
                 Destroy(nearbyInteractables[i].itemInteractionButton.gameObject);
                 nearbyInteractables.RemoveAt(i);
             }
         }
 
         Collider2D[] nearbyColliders = Physics2D.OverlapCircleAll(transform.position, interactionRange, interactableLayers);
         foreach (Collider2D collider in nearbyColliders)
         {
             ClientInteractable clientInteractable = collider.GetComponentInParent<ClientInteractable>();
             if (clientInteractable != null)
             {
                 if (clientInteractable.enabled == true)
                 {
                     if (nearbyInteractables.Any(i => i.interactable == clientInteractable) == false)
                     {
                         ItemInteractionButton itemInteractionButton = uiManager.interactionDisplay.AddItemButton(clientInteractable);
                         itemInteractionButton.button.onClick.AddListener(delegate
                         {
                             Interact(clientInteractable);
                         });
                         nearbyInteractables.Add(new NearbyInteractable(clientInteractable, itemInteractionButton));
                     }
                 }
             }
         }
         uiManager.interactionDisplay.ShowItemDisplay(nearbyInteractables.Count > 0);
     }
 
 private void Interact(ClientInteractable clientInteractable)
     {
         if (clientInteractable == null) return;
         LinkedEntityComponent linkedEntityComponent = clientInteractable.GetComponentInParent<LinkedEntityComponent>();
         clientInteractionComponentWriter.SendInteractEvent(new InteractionRequest(linkedEntityComponent.EntityId));
     }
 
     public class NearbyInteractable
     {
         public ClientInteractable interactable;
         public ItemInteractionButton itemInteractionButton;
 
         public NearbyInteractable(ClientInteractable clientInteractable, ItemInteractionButton button)
         {
             interactable = clientInteractable;
             this.itemInteractionButton = button;
         }
     }

Where uiManager just has this subclass:

 [System.Serializable]
     public class InteractionDisplay
     {
         public GameObject itemPanel, interactionPanel;
 
         public GameObject itemInteractionButtonPrefab;
         public Transform itemPanelContent;
 
         public ItemInteractionButton AddItemButton(ClientInteractable clientInteractable)
         {
             ItemInteractionButton itemInteractionButton = Instantiate(itemInteractionButtonPrefab, itemPanelContent).GetComponent<ItemInteractionButton>();
             itemInteractionButton.itemIcon.sprite = clientInteractable.interactionIcon;
             itemInteractionButton.itemText.text = clientInteractable.interactionMessage;
             return itemInteractionButton;
         }
 
         public void ShowItemDisplay(bool show)
         {
             if (itemPanel.activeSelf != show)
             {
                 itemPanel.SetActive(show);
             }
         }
 
         public void ShowDisplay(bool show)
         {
             if (interactionPanel.activeSelf != show)
             {
                 interactionPanel.SetActive(show);
             }
         }
     }