Unexpected Camera Rotation

Hey all, I've been writing a custom third person controller and I almost have the camera working the way I want for basic movement. The problem I am having is that whenever I rotate the camera with the right stick, and then move the player causing the camera to follow the player, when I go to rotate the camera again the camera tries to rotate to its last known rotation spot. I hope I worded that well enough. I have added my camera script and will provide screenshots if necessary.

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 
 public class CameraFromTutorial : MonoBehaviour {
 
     public float X_ANGLE_MIN, X_ANGLE_MAX;
     public bool cameraIsMovingByInput = false;
     [SerializeField]
     private float distanceAway, distanceUp;
     [SerializeField]
     private float camSmoothDampTime = 0.1f;
     [SerializeField]
     private Transform followTransform;
     [SerializeField]
     private Vector3 offset = new Vector3(0f, 1.5f, 0f);
     
     private Vector3 targetPosition;
     private Vector3 lookDir;
     private Vector3 velocityCamSmooth = Vector3.zero;
     private Vector3 input = Vector3.zero;
     private Vector3 cameraRotatePosition;
 
     // Use this for initialization
     void Start () {
         // Follow Transform is the player
         followTransform = followTransform.transform;
     }
     
     // Update is called once per frame
     void Update () {
         TrackInput ();
     }
 
     void LateUpdate () {
         // If the player isnt moving the right stick
         if (!cameraIsMovingByInput) {
             CameraFollow();
         } else {
             MoveCamera();
         }
 
         // Always Look at the player
         transform.LookAt(followTransform);
     }
 
     void TrackInput () {
         // This will just get the input values
         input.x = Input.GetAxis ("RHorizontal");
         input.y = Input.GetAxis  ("RVertical");
         input.z = 0f;
         // Set cameraIsMovingByInput to True if input has value else false
         if(input != Vector3.zero) {
             cameraIsMovingByInput = true;
         } else { 
             cameraIsMovingByInput = false;
         }
 
         if (input.sqrMagnitude > 1f) {
             input.Normalize ();
         }
 
         // We add this to the cameraInputPosition to know where to rotate to
         cameraRotatePosition.x += input.y;
         cameraRotatePosition.y += input.x;
 
         // Clamp the x so we cant rotate to above the players head or below his feet
         cameraRotatePosition.x = Mathf.Clamp(cameraRotatePosition.x, X_ANGLE_MIN, X_ANGLE_MAX);
     }
 
     void MoveCamera () {
         // Set the rotation relative to the input
         Quaternion rotation = Quaternion.Euler(cameraRotatePosition.x, cameraRotatePosition.y, 0);
         transform.position = Vector3.Lerp(transform.position, (followTransform.position + rotation * new Vector3(0f, 0f, -distanceAway)), Time.deltaTime * 3);
         Debug.DrawRay(transform.position, transform.forward * 3, Color.cyan, 1f);
     }
 
     void CameraFollow () {
         Vector3 characterOffset = followTransform.position + offset;
 
         lookDir = characterOffset - transform.position;
         lookDir.y = 0;
         lookDir.Normalize();
         Debug.DrawRay(transform.position, lookDir, Color.green);
 
         // Setting the target position by moving up by follow.up * distanceUp and back by follow.forward * distanceAway 
         targetPosition = characterOffset + followTransform.up * distanceUp - lookDir * distanceAway;
 
         SmoothPosition(transform.position, targetPosition);
     }
 
     private void SmoothPosition(Vector3 fromPos, Vector3 toPos) {
         transform.position = Vector3.SmoothDamp(fromPos, toPos, ref velocityCamSmooth, camSmoothDampTime);
     }
 }