Why does the camera smoothly follow in one direction but not the other?

Hello! I am currently trying to write a camera script for a 2D platformer. Everything works pretty good except for one thing: the camera will track the player smoothly and lerp to the correct position in one direction but not the other. The problem seems to be with trying to add an offset (the offsetLeft variable in the script) to the position the camera is supposed to lerp to. Depending on the sign of the offset value the camera will only work in one direction. In the other direction the camera will follow but it is very jittery and does not lerp to the position it is supposed to. After banging my head against the keyboard for hours trying to come up with a solution, I have given up and decided to ask the good people in the unity community for help. Why would it work perfectly in one direction and not the other? Any help will be greatly appreciated.

The script below is attached to the main camera in the scene.

 using UnityEngine;
 using System.Collections;
 
 public class ProtoCameraOne : MonoBehaviour {
 
     public Transform target;
 
     public float offsetLeft = 2f;
     public float offsetRight = 2f;
     public float offsetAbove = 2f;
     public float offsetBelow = 2f;
 
     public Vector2 margin, smoothing;
 
     public BoxCollider2D levelBounds;
 
     public bool isFollowing { get; set; }
 
     private Vector3 _min, _max;
     private float xDifLeft;
     private float xDifRight;
     private float yDifAbove;
     private float yDifBelow;
 
     GameObject player;
     ProtoMovement movement;
     Rigidbody2D rb;
 
     public void Start()
     {
         player = GameObject.FindGameObjectWithTag("Player");
         movement = player.GetComponent<ProtoMovement>();
         rb = player.GetComponent<Rigidbody2D>();
         _min = levelBounds.bounds.min;
         _max = levelBounds.bounds.max;
         isFollowing = true;
     }
 
     public void Update()
     {
         var x = transform.position.x;
         var y = transform.position.y;
 
         xDifLeft = x - target.position.x;
         xDifRight = target.position.x - x;
 
         yDifBelow = y - target.position.y;
         yDifAbove = target.position.y - y;
 
         var camOffsetValueX = target.position.x + offsetLeft;
 
 
         if(isFollowing)
         {
             if(Mathf.Abs(x - target.position.x) > margin.x)
             {
                 x = Mathf.Lerp(x, camOffsetValueX, smoothing.x * Time.deltaTime);
             }
 
             if(Mathf.Abs(y - target.position.y) > margin.y)
             {
                 y = Mathf.Lerp(y, target.position.y, smoothing.y * Time.deltaTime);
             }
         }
 
         var cameraHalfWidth = Camera.main.orthographicSize * ((float) Screen.width / Screen.height);
 
         x =  Mathf.Clamp(x, _min.x + cameraHalfWidth, _max.x - cameraHalfWidth);
         y =  Mathf.Clamp(y, _min.y + Camera.main.orthographicSize, _max.y - Camera.main.orthographicSize);
 
         transform.position = new Vector3(x, y, transform.position.z);
     }
 
 }