Different jump height

When player hit jump button i'm moving CharacterController upwards with .Move. Great, it works. But i gave player ability to jump higher when he's holding button down. And that's where things get strange. When i'm holding button down character jump at to heights randomly - one is little bit higher than the basic jump another high as planned.

 public class New_Player_Controller : MonoBehaviour {
 
     public float speed = 6.0f;
     public float jumpSpeed = 8.0f;
     public float gravity = 20.0f;
     public float rotateSpeedGrounded = 3.0f; // Rotation speed at ground
 
     public float inAirSpeed = 2.0f; // Adds or substracts to the movementspeed in air
     public float inAirDrift = 0.5f; // Adds Sidewards drift in air
     public float inAirRotation = 0.5f; // Rotation speed in air. Uncomment this to let the player change the face direction in air
 
     public float extraJumpForce = 5.0f;         //How much extra force to give to our jump when the button is held down
     public float maxExtraJumpTime = 2.0f;       //Maximum amount of time the jump button can be held down for
     public float delayToExtraJumpForce = 2.0f;  //Delay in how long before the extra force is added
     private float jumpTimer;             //Used in calculating the extra jump delay
     private bool playerJumped = false;           //Tell us if the player has jumped
     private bool playerJumping = false;          //Tell us if the player is holding down the jump button
 
     private Vector3 moveDirection = Vector3.zero;
     private float inputModifyFactor = 1.0f;
     private bool isGrounded = false;
     public Transform groundChecker;
     //public float rayDistance = 1.0f;
     public float playerWidth = 0.18f;
     private CharacterController controller;
 
     void Update() {
         //CharacterController controller = GetComponent<CharacterController> ();
         controller = GetComponent<CharacterController> ();
 
         float rayDistance = controller.bounds.extents.y;
         isGrounded = Physics.Raycast (transform.position, Vector3.down, rayDistance + 0.085f);
 
         //If our player hit the jump key, then it's true that we jumped!
         if (Input.GetButtonDown("Jump")) {
             //Debug.Log("pressed");
             if (isGrounded) {
                 playerJumped = true;   //Our player jumped!
                 playerJumping = true;  //Our player is jumping!
                 jumpTimer = Time.time; //Set the time at which we jumped
                 //Debug.Log("Jump");
             }
         }
 
         //If our player lets go of the Jump button OR if our jump was held down to the maximum amount...
         if (Input.GetButtonUp("Jump") || Time.time - jumpTimer > maxExtraJumpTime){
         //if (playerJumped || Time.time - jumpTimer > maxExtraJumpTime){
             playerJumping = false; //... then set PlayerJumping to false
         }
 
 // ---------------------------------------- At Ground -----------------------------------------
         if (isGrounded) {
             moveDirection = new Vector3 (Input.GetAxis ("Horizontal"), 0, Input.GetAxis ("Vertical"));
             moveDirection *= speed;
             inputModifyFactor = (moveDirection.x != 0.0f && moveDirection.z != 0.0f) ? 0.7071f : 1.0f;
             moveDirection *= inputModifyFactor;
 
             //if (Input.GetButtonDown ("Jump")) {
             if (playerJumped) {
                 moveDirection.y = jumpSpeed;
                 playerJumped = false;
             }
         }
         // ---------------------------------------------------- In Air  ----------------------------
         else {
             // We are not grounded. We can still influence the movement with the horizontal and vertical axis, but not so strong.
             moveDirection = new Vector3 (Input.GetAxis ("Horizontal"), moveDirection.y, Input.GetAxis ("Vertical"));
             moveDirection.x *= speed;
             moveDirection.z *= speed;
             moveDirection.y -= gravity * Time.deltaTime;
         }
         // ---------------------------------------------------------------------------------------------        
         //Rotate
         Vector3 curDir = new Vector3 (moveDirection.x, 0, moveDirection.z);
         Vector3 newDir = Vector3.RotateTowards (transform.forward, curDir, (controller.isGrounded ? rotateSpeedGrounded : inAirRotation) * Time.deltaTime, 0.0F);
         Debug.DrawRay (transform.position, newDir * 10, Color.red);
         transform.rotation = Quaternion.LookRotation (newDir);
     
          Move the controller
         if (moveDirection != Vector3.zero) {
 
 
             //Debug.Log (playerJumping);
             Debug.Log (playerJumping && Time.time - jumpTimer > delayToExtraJumpForce);
             //If our player is holding the jump button and a little bit of time has passed...
             if (playerJumping && Time.time - jumpTimer > delayToExtraJumpForce){
                 moveDirection.y = moveDirection.y + extraJumpForce;
             }
                 
             controller.Move (moveDirection * Time.deltaTime);
         }
     }

I don't understand. I'm not using physics (and i don't want to) so FixedUpdate will not (and it's not) help. Can you give me some advice?