Problem with collisions of my grab object

Hi! i have a object that can be grabbed, but if this object hit with other objects when i'm grabbing it, it move out of control. How i can make this object, remains in my hands when hit with other objects, thanks :)

 public class PickUpObjects : MonoBehaviour
 {
     public GameObject ObjectToPickUp;
     GameObject PickedObject;
     public Transform destination;
 
     void Update()
     {
         if (ObjectToPickUp != null && ObjectToPickUp.GetComponent<PickableObject>().isPickable == true && PickedObject == null)
         {
             if (Input.GetMouseButtonDown(0))
             {
                 PickedObject = ObjectToPickUp;
                 PickedObject.GetComponent<PickableObject>().isPickable = false;
                 PickedObject.transform.SetParent(destination);
                 PickedObject.transform.position = destination.position;
                 PickedObject.GetComponent<Rigidbody>().useGravity = false;                
             }
         }
         else if (PickedObject != null)
         {
             if (Input.GetMouseButtonUp(0))
             {
                 PickedObject.GetComponent<PickableObject>().isPickable = true;
                 PickedObject.transform.SetParent(null);
                 PickedObject.GetComponent<Rigidbody>().useGravity = true;                
                 PickedObject = null;
             }
         }
     }
 }
 
 -----------------------------------------------------------------------------------------------

 public class PickableObject : MonoBehaviour
 {
     private bool isPickable = true;
 
     void OnTriggerEnter(Collider other)
     {
         if (other.tag =="Destination")
         {
             other.GetComponentInParent<PickUpObjects>().ObjectToPickUp = this.gameObject;
         }
     }
 
     void OnTriggerExit(Collider other)
     {
         if (other.tag == "Destination")
         {
             other.GetComponentInParent<PickUpObjects>().ObjectToPickUp = null;
         }
 
     }
 }