RTS Camera: Rotate Around Point but then Pan Directions Flip

So Im trying to make an RTS Camera. Ive figured this out before but cant seem to now. Its a regular rts type camera but as the title says when I rotate the panning is mixed up. The panning uses an empty gameobject's forward to tell forward, back, left and right. Since I rotated about it, the game objects forward vector relative to the camera doesnt properly translate.

Heres what I have so far using UnityEngine; using System.Collections;

 public class CameraRTS : MonoBehaviour {
 
     public Transform cameraJoint;
 
     private Vector2 pos; //mouse
     public float panSensitvity = 0.25f; //sensitivity 0 to 1
     public float verticalSpeed = 1;
     public float cameraRotateSpeed = 80f;
     public float cameraDistance = 30f;
     
 
     //zoom stuff
     float ZoomAmount = 0;
     float MaxToClamp = 3;
     float ROTSpeed = 5;
     //attach this to camera, parent it to a gameObject
     // Use this for initialization
     void Start () {
         //transform.position = (0,1,-3.5);
         transform.localPosition = new Vector3 (0f, 1f, -3.5f);
         transform.localRotation = Quaternion.Euler(45f,0f,0f);
     }
     
     // Update is called once per frame
     void Update () {
 
         //get mouse 2d pos on screen
         pos = Camera.main.ScreenToViewportPoint(Input.mousePosition);
 
         //scroll and zoom
         ZoomAmount += Input.GetAxis("Mouse ScrollWheel");
         ZoomAmount = Mathf.Clamp(ZoomAmount, -MaxToClamp, MaxToClamp);
         var translate = Mathf.Min(Mathf.Abs(Input.GetAxis("Mouse ScrollWheel")), MaxToClamp - Mathf.Abs(ZoomAmount));
 
         transform.Translate(0,0,translate * ROTSpeed * Mathf.Sign(Input.GetAxis("Mouse ScrollWheel")));
 
 
         //rotation
         if (Input.GetKey (KeyCode.Mouse2)) {
             //need to lock cursor and make invisible
             if (pos.x >= 0.9f)
             {
                 //rotate left
                 transform.RotateAround(cameraJoint.position, Vector3.up, cameraRotateSpeed * Time.deltaTime);
 
             }
             if (pos.x <= 0.1f){
                 //rotate right
                 transform.RotateAround(cameraJoint.position, Vector3.down, cameraRotateSpeed * Time.deltaTime);
             }
             //pos.x;
 
         } else {
             mouseborder ();
         }
 
     }
 
     void mouseborder(){
         //mouse is in the far right of the screen
         if (pos.x >= 0.9f)
         {
             cameraJoint.transform.Translate(Vector3.right * panSensitvity);
         }
         //mouse is in the far left of the screen
         else if (pos.x <= 0.1f)
         {
             //move left
             cameraJoint.transform.Translate(Vector3.left * panSensitvity);
         }
         //mouse is in the middle of the screen in the horizontal axis
         else
         {
             //dont move in the horizontal axis
         }
         
         //mouse is in the far top of the screen
         if (pos.y >= 0.9f)
         {
             cameraJoint.transform.Translate(Vector3.forward * pos.y * panSensitvity);
 
         }
         //mouse is in the far bottom of the screen
         else if (pos.y <= 0.1f)
         {
             cameraJoint.transform.Translate(Vector3.back * panSensitvity);
         }
         //mouse is in the middle of the screen in the vertical axis
         else
         {
             //dont move in the vertical axis
         }
         
     }
 
 }