Unity Character Controller messing up.

I'm trying to make a click-to-move system, but it seems like whenever I press somewhere on the map, the character goes to the place, but doesn't stop, yet goes crazy. (Moving around in circles etc.)

If I increase the number in the vector3.Distance, it'll eventually stop going crazy, but the position will be far off.

 // Current selected player
     public GameObject curPlayer;
     CharacterController controller;
 
     public float speed = 5.0f;
     public Vector3 position;
 
     void Start ()
     {
         controller = curPlayer.GetComponent<CharacterController>();
 
         position = curPlayer.transform.position;
     }
 
     void Update()
     {
         if (GameManager.gameRunning) 
         {
             if (Input.GetMouseButtonUp(1))
             {
                 controller = curPlayer.GetComponent<CharacterController>();
 
                 locatePosition();
             }
             moveToPosition();
         }
     }
 
     void locatePosition()
     {
         RaycastHit hit;
         Ray ray = Camera.main.ScreenPointToRay (Input.mousePosition);
 
         if (Physics.Raycast(ray, out hit))
         {
             position = hit.point;
         }
     }
 
     void moveToPosition()
     {
         if (Vector3.Distance(curPlayer.transform.position, position) > 1)
         {
             Quaternion newRotation = Quaternion.LookRotation(position-curPlayer.transform.position, Vector3.forward);
 
             newRotation.x = 0f;
             newRotation.z = 0f;
 
             curPlayer.transform.rotation = Quaternion.Slerp (curPlayer.transform.rotation, newRotation, Time.deltaTime * 10);
             controller.SimpleMove(curPlayer.transform.forward * speed);
         }
     }