Separating smoothDampTime into separate axis

The smoothDamp script delays the camera interpolation when following the player, however I want there to be no delay when jumping and moving forward/backward, how would I add that? I have tried scirpting cam transform.pos.y = follow transform.pos.y but I couldn't get it to compile. Halp?

CAMERA SCRIPT:

  using UnityEngine;
             using System.Collections;
             
             public class CC_CameraTwo : MonoBehaviour {
             
             
                 
                 public float distanceAway = 5f;
                 public float distanceUp = 1f;
             
             
                 private Transform follow;
                 private Vector3 cameraTargetPosition;
             
             
                 [SerializeField]
                 private Vector3 camSmoothVelocity = Vector3.zero;
                 [SerializeField]
                 private float camSmoothDampTime = 3f;
             
             
                 // Use this for initialization
                 void Start () 
                 { 
                     follow = GameObject.FindGameObjectWithTag ("Follow").transform; 
                 }
             
                 
                 void LateUpdate ()     
                 {
                     // ** Camera distance from Player **
                     cameraTargetPosition = follow.position + Vector3.up * distanceUp - follow.forward * distanceAway;
             
                     // Camera smoothing 
                     smoothCam (this.transform.position, cameraTargetPosition);
                     transform.LookAt (follow);
                 }
             
             
                 // A private section to calculate the smoothCam variable
                 private void smoothCam (Vector3 fromPos, Vector3 toPos)
             
                 {
                     // ** Camera position transition Smoothing **
                     
                     this.transform.position = Vector3.SmoothDamp (fromPos, toPos, ref camSmoothVelocity, camSmoothDampTime);
                 
                 }
                     
             }
 

CharacterController script for reference:

 using UnityEngine;
 using System.Collections;
 
 public class CC_Logic : MonoBehaviour 
 
 {
     
     [SerializeField]
     private CharacterController controller;
 
 
     public float smooth = 3f;
     public float deadzone = 0.25f;
     public float turnSpeed = 800.0f;
     public float jumpForce = 3.0f;
     public float gravity = 10.0f;
     public float speed = 3.0f;
     public bool jump = false;
     
 
     private Vector3 moveDir = Vector3.zero;
     private float lockRotation = 0.0f;
 
     
     void Start ()
 
     {
         controller = GetComponent<CharacterController>();
     }
 
 
     void Update() 
     
     {
 
         // Is the controller on the ground?
         if (controller.isGrounded) 
 
         {
             jump = false;
 
 
             // Assigning Translation Controls
             moveDir = new Vector3(Input.GetAxis("Horizontal") , 0, Input.GetAxis("Vertical"));
 
 
             // Align forward vector of movement and camera. Turn it off when a camera isn't assigned. 
             moveDir = Camera.main.transform.rotation * moveDir;
 
 
             // Rotation parameters
             if(moveDir.sqrMagnitude > deadzone) 
             { transform.rotation = Quaternion.LookRotation(moveDir); }
 
 
             //Jump
             do if (Input.GetButtonDown("Jump"))
             {
                 moveDir.y = jumpForce;
                 jump = true;
             }
             while (jump = false);
 
         }
 
             // Applying gravity variable to the character controller when jumping
                moveDir.y -= gravity * Time.deltaTime;
 
             // Applying move direction to the character controller
                controller.Move(moveDir * speed * Time.deltaTime);
 
     }
 
     void LateUpdate()
     
     {
         // Lock rotations x and z to stop tilting. 
         controller.transform.rotation = Quaternion.Euler(lockRotation, transform.rotation.eulerAngles.y, lockRotation);
     }
 
 }