Character Controller behaves differently when a Ridigbody is in the scene

Hello and thank you in advance for helping me out,

i have a weird bug where the Speed at which my character is moved, seems to change a little bit as soon as there is a Rigidbody in the scene. This bug still occurs when i run a built.

I am moving my Character in FixedUpdate() with with Time.fixedDeltaTime and Time.DeltaTime where appropriate.

 public class Controller : MonoBehaviour
 {
     public Transform playerTransform;
     public Camera cameraPlayer;
     public CharacterController characterController;
     public string[] walkTags;
 
     public float mouseSensitivity = 100f;
     public float gravity = -9.1f;
 
     public float jumpHeight = 1.7f;
     public float moveAcceleration = 50;
     public float moveSpeed = 10;
     public float constantDrag = 40;
 
     private float mouseX;
     private float mouseY;
 
     private float xRotation;
 
     public Vector3 velocity;
 
     protected bool jump = false;
     private bool isOnGround = false;
 
     private Vector2 moveDirection;
 
     void Start()
     {
         xRotation = playerTransform.rotation.x;
         Cursor.lockState = CursorLockMode.Locked;
     }
 
 
     void Update()
     {
 
         if (isOnGround && Input.GetKey("space"))
         {
             jump = true;
         }
 
         mouseX += Input.GetAxis("Mouse X") * mouseSensitivity * Time.deltaTime;
         mouseY += Input.GetAxis("Mouse Y") * mouseSensitivity * Time.deltaTime;
 
         moveDirection.x = Input.GetAxis("Horizontal");
         moveDirection.y = Input.GetAxis("Vertical");
 
         moveDirection = Vector2.ClampMagnitude(moveDirection, 1);
     }
 
 
 
     void FixedUpdate()
     {
         if (jump)
         {
             velocity.y += Mathf.Sqrt(jumpHeight * -2 * gravity);
             jump = false;
         }
 
         //rotate camera and player
         xRotation -= mouseY;
         xRotation = Mathf.Clamp(xRotation, -90f, 90f);
         cameraPlayer.transform.localRotation = Quaternion.Euler(xRotation, 0f, 0f);
         playerTransform.Rotate(Vector3.up * mouseX);
 
         mouseX = 0;
         mouseY = 0;
 
 
         //apply movement
         float verticalVelocity = velocity.y;
         velocity.y = 0;
 
 
         velocity += transform.forward * moveDirection.y * moveAcceleration * Time.fixedDeltaTime
                         + transform.right * moveDirection.x * moveAcceleration * Time.fixedDeltaTime;
 
         velocity = Vector3.ClampMagnitude(velocity, moveSpeed);
 
 
         velocity.y = verticalVelocity;
 
 
         //apply gravity
         velocity += transform.up * gravity * Time.fixedDeltaTime;
 
         //apply drag
         verticalVelocity = velocity.y;
         velocity.y = 0;
 
         float signX = Mathf.Sign(velocity.x);
         float signZ = Mathf.Sign(velocity.z);
 
 
         velocity -= constantDrag * Time.fixedDeltaTime * velocity.normalized;
 
 
 
         if (signX != Mathf.Sign(velocity.x))
             velocity.x = 0;
 
         if (signZ != Mathf.Sign(velocity.z))
             velocity.z = 0;
 
 
         velocity.y = verticalVelocity;
 
 
         //move character
         isOnGround = false;
 
         characterController.Move(velocity * Time.fixedDeltaTime);
     }
 
 
     void OnControllerColliderHit(ControllerColliderHit hit)
     {
         foreach (string tag in walkTags)
         {
             if (string.Equals(hit.collider.tag, tag))
             {
                 velocity = Vector3.ProjectOnPlane(velocity, hit.normal);
 
                 if ((characterController.collisionFlags & CollisionFlags.Below) != 0)
                 {
                     isOnGround = true;
                 }
             }
         }
     }
 }