Player starts flying when colliding with Terrain

Hello. I am using the Character Controller to Move my player and somehow, if i jump or run against terrain my player gets stuck and is not falling back on the ground again, even if he is not grounded. I have looked this up in the internet for the past days, but haven't found a solution.

Also I am using Mirror, if it has anything to do with that...

Here is a video of my problem: https://www.youtube.com/watch?v=Ve7JbUOO59Q

Looking forward to seeing your creative solutions :)

Movement code:

 using UnityEngine;
 using Mirror;
 using Stranded.Inputs;
 
 public class PlayerMovementController : NetworkBehaviour
 {
     [SerializeField] private CharacterController controller;
     [SerializeField] private float movementSpeed = 5f;
     [SerializeField] private float jumpForce = 2f;
 
     private Vector3 moveDirection = Vector3.zero;
     private Vector2 previousInput;
 
     private Controls controls;
 
     private Animator anim;
 
     public float gravity = 9.81f;
 
     bool jumping = false;
     bool isGrounded = false;
 
     private Controls Controls
     {
         get
         {
             if (controls != null)
             {
                 return controls;
             }
             return controls = new Controls();
         }
     }
 
     public override void OnStartAuthority()
     {
         enabled = true;
         anim = gameObject.GetComponent<Animator>();
 
         //New Input System actions
         Controls.Player.Move.performed += ctx => SetMovement(ctx.ReadValue<Vector2>());
         Controls.Player.Move.canceled += ctx => ResetMovement();
         Controls.Player.Jump.performed += ctx => Jump();
         Controls.Player.Jump.canceled += ctx => JumpCancel();
     }
 
     [ClientCallback]
 
     private void OnEnable()
     {
         Controls.Enable();
     }
     [ClientCallback]
     private void OnDisable()
     {
         Controls.Disable();
     }
     [ClientCallback]
     private void Update()
     {
         Move();
     }
 
     [Client]
     //Vector2 of vertical/horizontal movement gets set
     private void SetMovement(Vector2 movement) => previousInput = movement;
 
     [Client]
 
     private void ResetMovement()
     {
         //reset Vector2 of vertical/horizontal movement
         previousInput = Vector2.zero;
     }
 
     [Client]
     private void Jump()
     {
         jumping = true;
     }
 
     [Client]
     private void JumpCancel()
     {
         jumping = false;
     }
 
     [Client]
 
     private void Move()
     {
         isGrounded = IsGrounded();
         if (isGrounded)
         {
             
             if(previousInput.y == 1)
             {
                 //Forward animation
                 anim.SetInteger("AnimationPar", 1);
             }
             else if(previousInput.y == -1)
             {
                 //Backwards animation
                 anim.SetInteger("AnimationPar", 2);
             }
             else if(previousInput.x == 0 && previousInput.y == 0 && jumping == false)
             {
                 //no inputs, idle animation
                 anim.SetInteger("AnimationPar", 0);
             }
             //Vector3 moveDirection depends on the inputs of previousInput
             moveDirection = new Vector3(previousInput.x, 0, previousInput.y);
             moveDirection = transform.TransformDirection(moveDirection);
 
             //Speed gets applied
             moveDirection *= movementSpeed;
 
             if(jumping == true)
             {
                 //jumping
                 moveDirection.y = jumpForce;
             }
         }
         else
         {
             //Player is not grounded anymore
 
             //Free fall animation
             anim.SetInteger("AnimationPar", 4);
 
             //Gavity gets applied
             moveDirection.y -= gravity * Time.deltaTime;
         }
 
         //Player gets moved by the Character Controller
         controller.Move(moveDirection * Time.deltaTime);
 
     }
 
     [Client]
     //Function to see, if the player is grounded
     private bool IsGrounded()
     {
         float ExtraHeightTest = 0.05f;
         //Raycast a Ray from the player towards the ground
         bool rayCastHit = Physics.Raycast(transform.position, Vector3.down, ExtraHeightTest);
         Color rayColor;
 
         if(rayCastHit == true)
         {
             //Player is grounded
             rayColor = Color.green;
         }
         else
         {
             //Player is not grounded
             rayColor = Color.red;
         }
 
         Debug.DrawRay(transform.position, Vector3.down * (transform.localScale.y / 2 + ExtraHeightTest), rayColor);
         return rayCastHit;
     }
 
 }