Camera Wall Collision is buggy

Hello guys,

I've been working on this problem since 5 hours but I cant figure out a good solution. So here is the problem: I have a script for collision detection with walls. It basically sets the camera position to a certain distance to the player, based on where a Ray hit happens. The problem is, that i want to set the distance to max distance if nothing is hit. What is happening now is that it is set everytime the currentDistance is adjusted, because then there is no hit. This is causing the camera to constantly jumping around.

Thanks for any help!

Here is my code:

     // Use this for initialization
     void Start()
     {
 
         cameraTransform = transform;
         cameraPosition = transform.position;
         cam = Camera.main;
         dollyDir = transform.localPosition.normalized;
 
         //        collision.Initialize(Camera.main);
         //        collision.UpdateCameraClipPoints(transform.position, transform.rotation, ref collision.adjustedCameraClipPoints);
         //        collision.UpdateCameraClipPoints(desiredCameraPos, transform.rotation, ref collision.desiredCameraClipPoints);
 
 
     }
 
     private void Update()
     {
 
         currentX += Input.GetAxis("Mouse X");
         currentY += Input.GetAxis("Mouse Y");
 
         currentY = Mathf.Clamp(currentY, Y_ANGLE_MIN, Y_ANGLE_MAX);
 
         desiredCameraPos = lookAt.TransformPoint(dollyDir * maxDistance);
         RaycastHit hit;
         RaycastHit backhit;
         float newdistance = 1f;
         Ray topLeft = Camera.main.ScreenPointToRay(new Vector3(0, 0, 0));
         Ray topRight = Camera.main.ScreenPointToRay(new Vector3(Screen.width, 0, 0));
         Ray botRight = Camera.main.ScreenPointToRay(new Vector3(Screen.width, Screen.height, 0));
         Ray botLeft = Camera.main.ScreenPointToRay(new Vector3(0, Screen.height, 0));
         //        if (Physics.Linecast(lookAt.position, desiredCameraPos, out hit))
         //        {
         //            currentDistance = Mathf.Clamp(hit.distance, minDistance, maxDistance);
         //        }
 
         Debug.DrawLine(lookAt.position, botRight.GetPoint(pointOnClippingRay), Color.cyan);
         Debug.DrawLine(lookAt.position, topRight.GetPoint(pointOnClippingRay), Color.cyan);
         Debug.DrawLine(lookAt.position, botLeft.GetPoint(pointOnClippingRay), Color.cyan);
         Debug.DrawLine(lookAt.position, topLeft.GetPoint(pointOnClippingRay), Color.cyan);
         
         if (Physics.Linecast(lookAt.position, botRight.GetPoint(pointOnClippingRay), out hit) || Physics.Linecast(lookAt.position, topRight.GetPoint(pointOnClippingRay), out hit) || Physics.Linecast(lookAt.position, topLeft.GetPoint(pointOnClippingRay), out hit) || Physics.Linecast(lookAt.position, botLeft.GetPoint(pointOnClippingRay), out hit))
         {
             newdistance = Mathf.Abs(hit.distance - currentDistance);
             currentDistance = Mathf.Clamp(hit.distance, minDistance, maxDistance);
         }
         //if(!Physics.Linecast(cameraTransform.position, -cameraTransform.forward, out hit, 1))
         //{
         //    currentDistance = maxDistance;
         //}
         else //if(!Physics.Raycast(cameraTransform.position, -cameraTransform.forward, out hit, 1) ))
             currentDistance = maxDistance;
 
    
     }
     // Update is called once per frame
     void LateUpdate()
     {
 
         Vector3 dir = new Vector3(0, 0, -currentDistance);
         Quaternion rotation = Quaternion.Euler(-currentY, currentX, 0);
         cameraTransform.position = lookAt.position + rotation * dir;
         cameraTransform.LookAt(lookAt.position);
 
         /*
         if(currentDistance < 2)
         {
             lookAtObject.GetComponent<Renderer>().material.color = new Color(1f, 0.92f, 0.016f, 0.3f);
         }
         */
     }