2D grab script collider problems

Hi, beginner here and I'm trying to use this script to simulate grabbing an object and throwing it.

 public class grabberscript : MonoBehaviour
 {
     private bool grabbed;
     RaycastHit2D hit;
     [SerializeField] private float distance = 2f;
     [SerializeField] private Transform holdpoint;
     [SerializeField] private float throwforce;
     private Quaternion originalRotation;
 
     void Update()
     {
 
         if (Input.GetKeyDown(KeyCode.E))
         {
             if (!grabbed)
             {
                 Physics2D.queriesStartInColliders = false;
 
                 hit = Physics2D.Raycast(transform.position, Vector2.right * transform.localScale.x, distance);
 
                 if (hit.collider != null && hit.collider.tag == "grabbable")
                 {
                     grabbed = true;    // Grabs object
                 }
 
             }
 
             else
             {
                 grabbed = false;
 
                 if (hit.collider.gameObject.GetComponent<Rigidbody2D>() != null)
                 {
                     hit.collider.gameObject.GetComponent<Rigidbody2D>().freezeRotation = false;
                     hit.collider.gameObject.GetComponent<Rigidbody2D>().velocity = new Vector2(transform.localScale.x, 1) * throwforce;
                 }
                 // throws object
             }
         }
 
         if (grabbed)
         {
             hit.collider.gameObject.transform.position = holdpoint.position;    // Move grabbed object to designated hold point
             hit.collider.gameObject.transform.rotation = originalRotation;    // Auto-rotate it back to original
             hit.collider.gameObject.GetComponent<Rigidbody2D>().freezeRotation = true;    // Freeze rotation
         }
 
     }
 
     void OnDrawGizmos() // Draw Raycast Distance
     {
         Gizmos.color = Color.green;
         Gizmos.DrawLine(transform.position, transform.position + Vector3.right * transform.localScale.x * distance);
     }
 }

My issue is that the box is able to clip and move through walls and other boxes while being grabbed. It is not able to clip through walls if it were to be pushed, however.