CharacterController doesn't jump properly while moving into platform

So I'm making a 3D platformer that uses some point-and-click movement. The idea is, the player object tries to keep moving towards the point that the player clicks, and the player can press a key to jump.

My issue is that when the player presses somewhere on a platform resting on the ground, the player object keeps trying to move into the platform. This wouldn't be an issue, except for some reason the jump doesn't work properly when this is happening. The player object jumps up a bit, then gets kind of stuck and falls back down.

Here's a video of the problem: https://vid.me/cZiL

Here's my code:

 using UnityEngine;
 using System.Collections;
 
 public class PlayerMove : MonoBehaviour {
     public float Speed;
     public float JumpSpeed;
     public float Gravity;
     
     private Vector3 destination;
 
     private CharacterController controller;
     private float yspeed;
 
     // Only check for Collision objects
     private int CollisionMask;
 
     void Start() {
         destination = transform.position;
         controller = GetComponent<CharacterController>();
         CollisionMask = 1 << LayerMask.NameToLayer("Collision");
     }
     
     private void SetDestination() {
         // Raycast to find where the user clicked
         RaycastHit hit;
         Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
         if (Physics.Raycast(ray, out hit, CollisionMask)) {
             destination = new Vector3(hit.point.x,
                                       0,
                                       hit.point.z);
         }
     }
     
     private void MoveToDestination() {
         // Move only in x and z
         destination.Set(destination.x,
                         transform.position.y,
                         destination.z);
         // If we're more than a given distance away, move closer
         if ((destination - transform.position).sqrMagnitude > Speed * Time.deltaTime) {
             controller.Move((destination - transform.position).normalized * Speed * Time.deltaTime);
         } else {
             transform.position = destination;
         }
     }
 
     private void Jump() {
         yspeed = JumpSpeed;
     }
     
     void Update() {
         if (controller.isGrounded) {
             // Jump if the player presses the appropriate key
             if (Input.GetKeyDown(KeyCode.Z)) {
                 Jump();
             }
         }
         yspeed -= Gravity * Time.deltaTime;
 
         // On left mouse click
         if (Input.GetMouseButtonDown(0)) {
             SetDestination();
         }
         MoveToDestination();
         
         controller.Move(Vector3.up * yspeed * Time.deltaTime);
     }
 }