Pan RTS camera controller without changing elevation/zoom

I would like to make an RTS camera controller with 3 functions:

1. Rotate if the RMB is held down or if Q and E are used

2. Pan forward/back/left/right using WSAD or if the mouse is pushed up against the walls of the screen. This should be done relative to the rotation, not the world coordinate system (i.e. forward should always be the direction your nose would point if you turned you head, not the direction your nose was pointed before turning).

3. Zoom if the scroll wheel is used

My current code doesn't quite work.... for some reason the camera sinks down to the min zoom level before panning backward and rises up to the max zoom level before panning forward. I want the zoom level to stay constant while panning and only change if the scroll wheel is used.

Here is the code:

 using UnityEngine;
 using System.Collections;
 
 public class CameraControls : MonoBehaviour {
     //public vaiables
     public float cameraPosX, cameraPosY, cameraPosZ;
     public float cameraRotX, cameraRotY, cameraRotZ;
     public float cameraSpeed = 30f, scrollSpeed = 30f, pixelEdge = 10f, minZoom = 10f, maxZoom = 30f;
 
     void Start () {
         //Sets transfrom.position/rotation to the values you set in the inspector.
         transform.position = new Vector3(cameraPosX,cameraPosY, cameraPosZ);
         transform.rotation = Quaternion.Euler(cameraRotX, cameraRotY, cameraRotZ);
     }
 
     void LateUpdate () {
         //Vector3 zoom variable.
         Vector3 zoom = transform.position;
         
         // Zooming if in zoom range, else holding constant zoom level
         if(zoom.y < minZoom){
             zoom.y = minZoom;
             transform.position = zoom;
         }
         if(zoom.y > maxZoom){
             zoom.y = maxZoom;
             transform.position = zoom;
         }
 
         // Panning if move buttons are pressed or if mouse is in pan zones on screen edge
         if(Input.mousePosition.x > Screen.width - pixelEdge && !Input.GetMouseButton(1)|| Input.GetKey(KeyCode.D) && !Input.GetMouseButton(1)){
             transform.position += transform.right * Time.deltaTime * cameraSpeed;
         }
         if(Input.mousePosition.x < 0 + pixelEdge && !Input.GetMouseButton(1) || Input.GetKey(KeyCode.A) && !Input.GetMouseButton(1)){
             transform.position += transform.right * Time.deltaTime * -cameraSpeed;
         }
         if(Input.mousePosition.y > Screen.height - pixelEdge && !Input.GetMouseButton(1) || Input.GetKey(KeyCode.W) && !Input.GetMouseButton(1)){
             transform.position += transform.up * Time.deltaTime * cameraSpeed;
         }
         if(Input.mousePosition.y < 0 + pixelEdge && !Input.GetMouseButton(1) || Input.GetKey(KeyCode.S) && !Input.GetMouseButton(1)){
             transform.position += transform.up * Time.deltaTime * -cameraSpeed;
         }
         
         //Rotates the camera if Q or E are pressed or if RMB is down and mouse pushes on screen edge
         if(Input.GetKey(KeyCode.Q)){
             transform.Rotate(0, -2.5f * Time.deltaTime * cameraSpeed, 0, Space.World);
         }
         if(Input.GetKey(KeyCode.E)){
             transform.Rotate(0, 2.5f * Time.deltaTime * cameraSpeed, 0, Space.World);
         }
         if(Input.mousePosition.x > Screen.width - pixelEdge && Input.GetMouseButton(1)){
             transform.Rotate(0, 2.5f * Time.deltaTime * cameraSpeed, 0, Space.World);
         }
         if(Input.mousePosition.x < 0 + pixelEdge && Input.GetMouseButton(1)){
             transform.Rotate(0, -2.5f * Time.deltaTime * cameraSpeed, 0, Space.World);
         }
         
         //Zooms the camera with the scroll wheel
         if(Input.GetAxis("Mouse ScrollWheel")< 0){
             if(zoom.y < maxZoom){
                 transform.Translate(Vector3.back * Time.deltaTime * scrollSpeed);
             }
         }
         if(Input.GetAxis("Mouse ScrollWheel") > 0){
             if(zoom.y > minZoom){
                 transform.Translate(Vector3.forward * Time.deltaTime * scrollSpeed);
             }
         }
 
     }
     
 }

I suspect this has to do with how I'm handling the zooming but I can't figure out what exactly has to change. Thoughts?