Unity ortographic camera panning within boundaries issue

I have an ortographic camera and I need to create panning with mouse within boundaries. I currently have this code which works fine with one problem. When user drags camera out of bounds, mouse position still gets updated and user needs to first move the mouse back within the bounds before camera is panning again. E.g. let's say the max bounds of x axis is 140. If the user moves the mouse to 150, the camera's position is clamped to 140 which is fine. However, if the user pans the camera back without releasing button, the camera is stuck at 140 until the mouse position reaches 140 again. Any ideas how to implement this adjustment?

 public class CameraDragMouse : MonoBehaviour
 { 
     private Vector3 Origin;
     private Vector3 Diference;
     private bool Drag;
 
     private float minPanningBoundaryX;
     private float maxPanningBoundaryX;
     private float minPanningBoundaryZ;
     private float maxPanningBoundaryZ;
 
     private Camera mainCamera;
 
     private void Start()
     {
         mainCamera = this.GetComponentInChildren<Camera>();
         CameraCenter.CenterCameraOnPosition(new Vector3(24, 0, 24));
         minPanningBoundaryX = gameObject.transform.position.x - 40;
         maxPanningBoundaryX = gameObject.transform.position.x + 40;
         minPanningBoundaryZ = gameObject.transform.position.z - 40;
         maxPanningBoundaryZ = gameObject.transform.position.z + 40;
     }
 
     void LateUpdate()
     {
         if (Input.GetMouseButton(0))
         {
             Ray rr = Camera.main.ScreenPointToRay(Input.mousePosition);
             if (Physics.Raycast(rr, out RaycastHit hit, Mathf.Infinity, Constants.LAYER_MASK_TERRAIN))
             { 
                 Diference = hit.point - gameObject.transform.position;
             }
 
             if (Drag == false)
             {
                 Drag = true;
                 Origin = hit.point;
             }
         }
         else
         {
             Drag = false;
         }
         if (Drag == true)
         {
             Vector3 newCameraPosition = Origin - Diference;
             newCameraPosition.x = Mathf.Clamp(newCameraPosition.x, minPanningBoundaryX, maxPanningBoundaryX);
             newCameraPosition.z = Mathf.Clamp(newCameraPosition.z, minPanningBoundaryZ, maxPanningBoundaryZ);
             gameObject.transform.position = newCameraPosition;
         }
     }