How to make Objects moving on a circular path collide with each other?

Hi I am currently working on a circular puzzle game with moving blocks

You can see my current progress also in this gif: https://media.giphy.com/media/SjOQCFVUoXgyaeFbdx/giphy.gif

I am struggling to figure out how to make them "collide". I tried using Colliders with RigidBodies but then i need to move the blocks with RigidBody2d.MovePosition and RigidBody2D.MoveRotation which makes moving them with the mouse very clunky, meaning they sometimes don't stay at the location i want them to and they jump around when moving the mouse to far away.

Moving them with transform looks much better and precise, which i need in my game. I tried using raycast at each side of the block and when they hit another block the block stops.. I tried clamping the angle between the middle and the block when a raycasts hits antoher block but that didnt work eiter. Is there an easy way to accomplish the behavior i want?

Here it the Code i use on the blocks to move them with the mouse:

 public class MoveableBlockCircular : MonoBehaviour
 {
     public float distanceFromCentre = 5.0f;
     public Transform centerTransform;
     public Transform blockTransform;
 
     private Plane centerControlPlane = new Plane();
     private Camera cam;
 
     private bool mouseDown;
 
     void Awake()
     {
         cam = Camera.main;
     }
 
     void OnMouseDown()
     {
         UpdateCentreControlPlane();
         UpdateBlockPosition();
         mouseDown = true;
     }
 
     void OnMouseUp()
     {
         mouseDown = false;
     }
 
 
     private void FixedUpdate()
     { 
         if (mouseDown)
         {
             UpdateBlockPosition();
         }
     }
 
     void UpdateCentreControlPlane()
     {
         Vector3 centrePos = centerTransform.position;
         centerControlPlane.Set3Points(centrePos + Vector3.left, centrePos + Vector3.up, centrePos + Vector3.right);
     }
 
     void UpdateBlockPosition()
     {
         Vector3 centrePosition = centerTransform.position;
         Vector3 posDifference = GetPlanePoint(Input.mousePosition) - centrePosition;
         posDifference = ClampToMagnitude(posDifference, distanceFromCentre, distanceFromCentre);
 
         blockTransform.position = centrePosition + posDifference;
         LookAt2D(blockTransform, centrePosition);        
     }
 
     Vector3 GetPlanePoint(Vector3 pos)
     {
         float distance;
 
         Ray ray = cam.ScreenPointToRay(pos);
         if (centerControlPlane.Raycast(ray, out distance))
         {
             return ray.GetPoint(distance);
         }
 
         return pos;
     }
 
     Vector3 ClampToMagnitude(Vector3 vec, float min, float max)
     {
         return vec.normalized * Mathf.Clamp(vec.magnitude, min, max);
     }
 
     public void LookAt2D(Transform transform, Vector2 target)
     {
         Vector2 current = transform.position;
         var direction = target - current;
         var angle = Mathf.Atan2(direction.y, direction.x) * Mathf.Rad2Deg;
 
         //with rigidBody
         //Quaternion quatRotation = Quaternion.AngleAxis(angle, Vector3.forward) * Quaternion.Euler(0, 0, 90);
         //rb.SetRotation(quatRotation.eulerAngles.z);
         //rb.MoveRotation(quatRotation.eulerAngles.z);
 
         //with transform
         transform.rotation = Quaternion.AngleAxis(angle, Vector3.forward) * Quaternion.Euler(0, 0, 90); //Magic
     }
 }