My player ball is shaking when it changes position

Hey guys my question is in the title. Basically when I press play everything is OK, the ball starts to go up without shaking, but when I start swiping to change position it starts shaking.

I already tried different things like changing to FixedUpdate() or reseting the player, but it doesn't change. Btw there is no animation on the ball. Can you help me ?

Here is the script with the swipe parameters :

 public class Swipe : MonoBehaviour 
 {
 private const float DEADZONE = 100.0f;

 public static Swipe Instance { set; get; }

 private bool tap, swipeLeft, swipeRight, swipeUp, swipeDown;
 private Vector2 startTouch, swipeDelta;

 public bool Tap { get { return tap; } }
 public Vector2 SwipeDelta { get { return swipeDelta; } }
 public bool SwipeLeft { get { return swipeLeft; } }
 public bool SwipeRight { get { return swipeRight; } }
 public bool SwipeUp { get { return swipeUp; } }
 public bool SwipeDown { get { return swipeDown; } }

 private void Awake()
 {
     Instance = this;
 }

 private void Update()
 {
     // Reseting all the booleans
     tap = swipeLeft = swipeRight = swipeDown = swipeUp = false;

     #region Stadalone Inputs
     if (Input.GetMouseButtonDown(0))
     {
         tap = true;
         startTouch = Input.mousePosition;
     }
     else if (Input.GetMouseButtonUp(0))
     {
         startTouch = swipeDelta = Vector2.zero;
     }
     #endregion

     #region Mobile Inputs
     if (Input.touches.Length != 0)
     {
         if (Input.touches[0].phase == TouchPhase.Began)
         {
             tap = true;
             startTouch = Input.mousePosition;
         }
         else if (Input.touches[0].phase == TouchPhase.Ended || Input.touches[0].phase == TouchPhase.Canceled)
         {
             startTouch = swipeDelta = Vector2.zero;
         }
     }
     #endregion

     // Calculate The Distance
     swipeDelta = Vector2.zero;
     if (startTouch != Vector2.zero)
     {
         if (Input.touches.Length != 0)
         {
             swipeDelta = Input.touches[0].position - startTouch;
         }
         else if (Input.GetMouseButton(0))
         {
             swipeDelta = (Vector2)Input.mousePosition - startTouch;
         }
     } 

     // Did we cross the deadzone ?
     if (swipeDelta.magnitude > DEADZONE)
     {
         // Which direction ?
         float x = swipeDelta.x;
         float y = swipeDelta.y;

         if (Mathf.Abs(x) > Mathf.Abs(y))
         {
             // Left or Right
             if (x < 0)
                 swipeLeft = true;
             else
                 swipeRight = true;
         }
         else
         {
             // Up or Down
             if (y < 0)
                 swipeDown = true;
             else
                 swipeUp = true;
         }

         startTouch = swipeDelta = Vector2.zero;
     }
 }   
 }

And this the part of the script in Update() or (FixedUpdate(), it doesn't change anything) on the player :

  // Gather the inputs in which tube we should be
     if (Swipe.Instance.SwipeLeft)
         MoveTube(false);
     if (Swipe.Instance.SwipeRight)
         MoveTube(true);


     // Calculate where we should be in the future
     Vector3 targetPosition = transform.position.y * Vector3.up;
     if (desiredTube == 0)
         targetPosition += Vector3.left * TUBE_DISTANCE;
     else if (desiredTube == 2)
         targetPosition += Vector3.right * TUBE_DISTANCE; 

     // Let's calculate our move delta
     Vector3 moveVector = Vector3.zero;
     moveVector.x = (targetPosition - transform.position).normalized.x * speed;
     moveVector.y = speed;

     // Move the ball
     controller.Move(moveVector * Time.deltaTime);

 }

 private void MoveTube(bool goingRight)
 {
     desiredTube += (goingRight) ? 1 : -1;
     desiredTube = Mathf.Clamp(desiredTube, 0, 2);
 }

Thanks.