Shake when moving the camera

Good day. I tried to make camera movement script. The idea is: user clicks on the plane, then dragging the mouse moving the camera (like in startegies or map editors). But I got the problem: when moving the camera it starting to shake. With more mouse displacement - more dynamic shaking. Here is my code:

 public class CameraHandler : MonoBehaviour {
 
    
     private Vector3 dragOrigin;
     private Vector3 dragDelta;
     private Vector3 camOrigin;
     private bool dragStarted = false;
     public Plane groundPlane;
     public Transform markerObject;
 
 
     
     void Start () {
         groundPlane = new Plane(Vector3.up, Vector3.zero);
     }
     
     
     void Update () {
         if (Input.GetMouseButton(0))
         {
             Vector3 planePos = new Vector3(0, 0, 0);
             Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
             float rayDistance;
             if (groundPlane.Raycast(ray, out rayDistance))
                 planePos = ray.GetPoint(rayDistance);
 
             if (!dragStarted)
             {               
                 dragOrigin = planePos;                
                 camOrigin = Camera.main.transform.position;
                 dragStarted = true;
                 return;
             }
             else
             {   
                 markerObject.position = planePos;
                 dragDelta = dragOrigin - planePos;
                 Camera.main.transform.position = camOrigin + dragDelta;
             }
         }        
         else if (!Input.GetMouseButton(0) && dragStarted)
         {           
 
             dragStarted = false;
             return;
         }        
     }
     
     
 }