My player collision is not accurate/precise enough

basically, whenever the player (pink cube) moves, it hits the collider but then only gets pushed back to this point (won't go further,i want it to like be closer to the collider if u know what i mean). It kind of improves if i change the speed but then it'd still be a problem for the other colliders. (and also not to mention it happens 100% of the time if i port it to android). Anyway to solve this? maybe my code for oncontrollercolliderhit is not accurate/good enough(i'm using charactercontroller). or maybe someone can suggest another way of moving my player which will work(i tried rigidbodies but idk what i did wrong it wouldn't work)

Here's basically the code responsible for everything

     public Swipe swipeControls;
     public Transform plr;
     private Vector3 desiredPosition;
     private Rigidbody rb;
     private CharacterController controller;
     private Vector3 startPos;
     private static int sceneCount;
     
     // Update is called once per frame
     private void Start()
     {
         desiredPosition = plr.transform.position;
         rb = plr.GetComponent<Rigidbody>();
         controller = plr.GetComponent<CharacterController>();
         startPos = plr.transform.position;
         sceneCount = SceneManager.GetActiveScene().buildIndex;
     }
     private void Update ()
     {
         if (swipeControls.SwipeLeft)
             desiredPosition += Vector3.left * 30;
             //plr.GetComponent<Rigidbody>().AddForce(Vector3.left);
             //plr.transform.Translate(-100f * Time.deltaTime, 0f, 0f);
         if (swipeControls.SwipeRight)
             desiredPosition += Vector3.right * 30;
              //plr.transform.Translate(100f * Time.deltaTime, 0f, 0f);
         if (swipeControls.SwipeUp)
             desiredPosition += Vector3.forward * 30;
         if (swipeControls.SwipeDown)
             desiredPosition += Vector3.back * 30;
         
 
         //plr.transform.position = Vector3.MoveTowards(plr.transform.position, desiredPosition,25f * Time.deltaTime);
         var dir = desiredPosition - plr.transform.position;
         var movement = dir.normalized * 21f * Time.deltaTime;
         var gravity = 9.8f;
         var gravityfps = 0f;
         gravityfps -= gravity * Time.deltaTime;
         movement.y = gravityfps;
         if (movement.magnitude > dir.magnitude)
         movement = dir;
 
         
         controller.Move(movement);
     }
 
 
 
     private void OnControllerColliderHit(ControllerColliderHit hit)
     {
         if(hit.gameObject.tag == "death")
         {
             Debug.Log("die");
             plr.transform.position = new Vector3(-2.66f, 2.017f, -10f);
             desiredPosition = plr.transform.position;
         }
         if(hit.gameObject.tag == "walls")
         {
             desiredPosition = plr.transform.position;
         }
         if(hit.gameObject.tag == "end")
         {
             sceneCount = SceneManager.GetActiveScene().buildIndex;
             sceneCount = sceneCount + 1;
             //PlayerPrefs.SetInt("sceneNo", sceneCount);
             SceneManager.LoadScene(sceneCount);
         }
         Debug.Log(sceneCount);
         Debug.Log(hit.collider.name);
     }
 }