Drag Rigidbody that respects collisions.

Hi there, i have a game where you drag blocks only across one axis, either the Z or the X. The problem im having at the minute is the movement isn't 1:1, the further away the block gets from the camera the slower it travels. I need it to be a constant speed that will also respect collisions, as moving to fast of course goes through other colliders.

Here is the current code

     public virtual void FixedUpdate()
     {
         
         if (!ReferenceManager.moveController.CanMove) { return; }
 
         //Debug.DrawRay(_rayPos.transform.position + new Vector3(raycastOffset.x, 0, raycastOffset.y), Vector3.down, Color.red);
 
 
         if (Input.GetMouseButton(0) && CanMove)
         {
             ray = Camera.main.ScreenPointToRay(Input.mousePosition);
             if (!dragObj && (Physics.Raycast(ray, out hit) && hit.rigidbody))
             {
                 if (!gameObject) return;    //check we're still there
                 dragObj = hit.transform;
                 Length = hit.distance;
                 Debug.Log(Length);
                 _offset = hit.point - dragObj.position;
 
                 _rigidbody.constraints = RigidbodyConstraints.FreezeAll;
                 _rigidbody.isKinematic = true;
                 _rigidbody.mass = TempMass;
               
                 if (dragObj.GetComponentInParent<BlockBehaviour>().blockType == BlockType.Horizontal)
                 {
                     dragObj.GetComponent<Rigidbody>().constraints = RigidbodyConstraints.FreezePositionY |
                                                                     RigidbodyConstraints.FreezePositionZ |
                                                                     RigidbodyConstraints.FreezeRotation;
                     dragObj.GetComponent<Rigidbody>().mass = CachedMass;
                     dragObj.GetComponent<Rigidbody>().isKinematic = false;
                 }
                 else
                 {
                     dragObj.GetComponent<Rigidbody>().constraints = RigidbodyConstraints.FreezePositionX |
                                                                     RigidbodyConstraints.FreezePositionY |
                                                                     RigidbodyConstraints.FreezeRotation;
                     dragObj.GetComponent<Rigidbody>().mass = CachedMass;
                     dragObj.GetComponent<Rigidbody>().isKinematic = false;
                 }
             }
         }
         else
         {
             dragObj = null;
         }
         if (!dragObj) return;
         if (dragObj == null) return;
 
 //        var wantedPos = Camera.main.ScreenToWorldPoint(new Vector3(Input.mousePosition.x, Input.mousePosition.y, 10)); 
 //        _rigidbody.MovePosition(new Vector3(wantedPos.x, transform.position.y, transform.position.z)) * Time.deltaTime;
 //        transform.position = new Vector3(wantedPos.x, transform.position.y, transform.position.z);
 
         var vel = (ray.GetPoint(Length) - (dragObj.position + _offset)) * Speed;
         if (vel.magnitude > MaxSpeed)
             vel *= MaxSpeed / vel.magnitude;
         dragObj.gameObject.GetComponentInChildren<Rigidbody>().velocity = vel;
     }