Limit Variable Orthographic Camera Movement

I have a 2D map and a orthographic camera. The player can move (transform.Translate) and zoom (orthographic.Size) the camera. I need the camera not to leave the map.

With GeometryUtility.CalculateFrustumPlanes, GeometryUtility.TestPlanesAABB and a collider on each side of the map I tried the following:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.EventSystems;
 
 public class CamMovement : MonoBehaviour {
 
     BoxCollider2D limitRight;
     BoxCollider2D limitLeft;
     BoxCollider2D limitUp;
     BoxCollider2D limitDown;
 
     Camera cam;
     Plane[] planes;
 
     Vector3 touchStart;
 
     [Header("Zoom Limits")]
     public float zoomMin;
     public float zoomMax;
 
     [Header("Zoom Speed")]
     public float scrollSpeed;
     public float pinchSpeed;
 
     bool recentlyZoom = false;
     bool touchedMap = false;
 
     float currentZoomMax;
 
     float cameraMinX = -10f;
     float cameraMaxX = 200f;
     float cameraMinY = -100f;
     float cameraMaxY = 10f;
 
     private void Start()
     {
         cam = Camera.main;
 
         limitRight = GameObject.Find("camera_limit_right").GetComponent<BoxCollider2D>();
         limitLeft = GameObject.Find("camera_limit_left").GetComponent<BoxCollider2D>();
         limitUp = GameObject.Find("camera_limit_up").GetComponent<BoxCollider2D>();
         limitDown = GameObject.Find("camera_limit_down").GetComponent<BoxCollider2D>();
     }
 
     void FixedUpdate()
     {
         SetLimits();
         Move();
     }
 
     //SetLimits
     private void SetLimits()
     {     
         planes = GeometryUtility.CalculateFrustumPlanes(cam);
 
         if (GeometryUtility.TestPlanesAABB(planes, limitLeft.bounds))
         {
             Debug.Log("limitLeft reached");
             currentZoomMax = cam.orthographicSize;
             cameraMinX = cam.transform.position.x;
         }
         else if (GeometryUtility.TestPlanesAABB(planes, limitRight.bounds))
         {
             Debug.Log("limitRight reached");
             currentZoomMax = cam.orthographicSize;
             cameraMaxX = cam.transform.position.x;
         }
         else if (GeometryUtility.TestPlanesAABB(planes, limitUp.bounds))
         {
             Debug.Log("limitUp reached");
             currentZoomMax = cam.orthographicSize;
             cameraMaxY = cam.transform.position.y;
         }
         else if (GeometryUtility.TestPlanesAABB(planes, limitDown.bounds))
         {
             Debug.Log("limitDown reached");
             currentZoomMax = cam.orthographicSize;
             cameraMinY = cam.transform.position.y;
         }
         else
         {
             currentZoomMax = zoomMax;
             cameraMinX = -10f;
             cameraMaxX = 200f;
             cameraMinY = -100f;
             cameraMaxY = 10f;
         } 
     }
 
     //Move
     void Move()
     {
         if (Input.GetMouseButtonDown(0))
         {
             if (!EventSystem.current.IsPointerOverGameObject())
             {
                 touchedMap = true;
             }
             else
             {
                 touchedMap = false;
             }
 
             foreach (Touch touch in Input.touches)
             {
                 int id = touch.fingerId;
                 if (!EventSystem.current.IsPointerOverGameObject())
                 {
                     touchedMap = true;
                 }
                 else
                 {
                     touchedMap = false;
                 }
             }
         }
 
         if (Input.GetMouseButtonDown(0) && touchedMap)
         {
             touchStart = cam.ScreenToWorldPoint(Input.mousePosition);
         }
 
         if (Input.touchCount == 2)
         {
             recentlyZoom = true;
 
             Touch touchZero = Input.GetTouch(0);
             Touch touchOne = Input.GetTouch(1);
 
             Vector2 touchZeroPrevPos = touchZero.position - touchZero.deltaPosition;
             Vector2 touchOnePrevPos = touchOne.position - touchOne.deltaPosition;
 
             float prevMagnitude = (touchZeroPrevPos - touchOnePrevPos).magnitude;
             float currentMagnitude = (touchZero.position - touchOne.position).magnitude;
 
             float difference = currentMagnitude - prevMagnitude;
 
             Zoom(difference * pinchSpeed);
         }
         else if (Input.GetMouseButton(0) && touchedMap)
         {
             if (recentlyZoom)
             {
                 touchStart = Camera.main.ScreenToWorldPoint(Input.mousePosition);
                 recentlyZoom = false;
             }
 
             Vector3 direction = touchStart - cam.ScreenToWorldPoint(Input.mousePosition);
 
             Camera.main.transform.Translate(direction, Space.Self);
 
             Camera.main.transform.position = new Vector3(
               Mathf.Clamp(cam.transform.position.x, cameraMinX, cameraMaxX),
               Mathf.Clamp(cam.transform.position.y, cameraMinY, cameraMaxY),
               -1);
         }
 
         if (Input.touchCount < 1)
         {
             recentlyZoom = false;
         }
 
         Zoom(Input.GetAxis("Mouse ScrollWheel") * scrollSpeed);
     }
 
     void Zoom(float increment)
     {
         cam.orthographicSize = Mathf.Clamp(cam.orthographicSize - increment, zoomMin, currentZoomMax);
     }
 }

The script works, the problem is that if I move the camera very quickly, it crosses the limit.