Brake on wheel collider "slips"

I have a vehicle using wheel colliders that drives left to right fixed along the X-Axis using Horizontal inputs. I have it scripted to auto-brake when an input key is released, causing it to stop instantly, which is what I want. But sometimes when accelerating with the right key and then switching quickly to accelerate with the left key, the vehicle will continue to slide to the right briefly before accelerating left.

At first I thought it had to do with tire friction/grip but I think it may be an Input problem. Anyone have a suggestion of what I can do to find out what's happening and how to fix it? Here is the script powering the vehicle in case it's useful:

 public class Car : MonoBehaviour {
     
     public Transform centerOfMass;
     
     public WheelCollider FrontLeftWheel;
     public WheelCollider FrontRightWheel;
     public WheelCollider BackLeftWheel;
     public WheelCollider BackRightWheel;
     
     public float[] GearRatio;
     public int CurrentGear = 0;
     
     public float EngineTorque = 230;
     public float MaxEngineRPM = 3000;
     public float MinEngineRPM = 1000;
     private float EngineRPM = 0;
     //public float BrakeTorque = 300;
 
     // Use this for initialization
     void Start () {
         
     if(centerOfMass != null)
         rigidbody.centerOfMass = centerOfMass.localPosition;
          
 
     }
     
      
     
     
     // Update is called once per frame
     void Update () {
         
         EngineRPM = (FrontLeftWheel.rpm + FrontRightWheel.rpm)/2 * GearRatio[CurrentGear];
         ShiftGears();
         
         audio.pitch = Mathf.Abs(EngineRPM / MaxEngineRPM) + 1.0f;
         if(audio.pitch > 2.0f) {
             audio.pitch = 2.0f;
         }
         
         FrontLeftWheel.motorTorque = EngineTorque / GearRatio[CurrentGear] * Input.GetAxis ("Horizontal");
         FrontRightWheel.motorTorque = EngineTorque / GearRatio[CurrentGear] * Input.GetAxis ("Horizontal");
         BackLeftWheel.motorTorque = EngineTorque / GearRatio[CurrentGear] * Input.GetAxis ("Horizontal");
         BackRightWheel.motorTorque = EngineTorque / GearRatio[CurrentGear] * Input.GetAxis ("Horizontal");
         
         
         
         if(Input.GetAxis ("Horizontal") == 0){
             FrontLeftWheel.brakeTorque = Mathf.Infinity;
             FrontRightWheel.brakeTorque = Mathf.Infinity;
             BackLeftWheel.brakeTorque = Mathf.Infinity;
             BackRightWheel.brakeTorque = Mathf.Infinity;
         }
         
         else {
             FrontLeftWheel.brakeTorque = 0;
             FrontRightWheel.brakeTorque = 0;
             BackLeftWheel.brakeTorque = 0;
             BackRightWheel.brakeTorque = 0;
         }
             
     
     }
     
     void ShiftGears() {
         
         int AppropriateGear;
         
         if(EngineRPM >= MaxEngineRPM){
             AppropriateGear = CurrentGear;
             
             for(int i = 0; i < GearRatio.Length; i ++){
                 if(FrontLeftWheel.rpm * GearRatio[i] < MaxEngineRPM){
                     AppropriateGear = i;
                     break;
                 }
             }
             CurrentGear = AppropriateGear;
         }
         
         if(EngineRPM <= MinEngineRPM){
             AppropriateGear = CurrentGear;
             
             for(int j = GearRatio.Length-1; j >= 0; j --){
                 if(FrontLeftWheel.rpm * GearRatio[j] > MinEngineRPM){
                     AppropriateGear = j;
                     break;
                 }
             }
             CurrentGear = AppropriateGear;
         }
         
             
         
     }    
 }