Problem with Holstering Sword in VR

I created a script where if one of the player's hand is in the holsters trigger and using the VRTK input script, they can holster and unholster the sword when pressing the grip button. It works fine however if let's say I unholstered the sword using my right hand and then got my left hand, placed it in the holster's trigger, then press the grip button on the left controller, I will be unable to holster the right sword unless my left hand is in the trigger and I press the right controller grip button. The same thing happens if I switch the hands. Is there any way to fix this?

The Script:

  public SettingManager settings;
     // Use this for initialization
     void Start () {
         SwordL.SetActive(false);
         SwordR.SetActive(false);
     }
 
     public bool LeftHolsterTouching;
     public bool LeftSwordTaken;
 
     public bool InLeftHand;
     public bool InRightHand;
 
     public GameObject HandL;
     public GameObject HandR;
 
     public GameObject SwordL;
     public GameObject SwordR;
 
     public void OnTriggerEnter(Collider other)
     {
         if(other.tag == "LeftHand" || other.tag == "SwordL" ||  other.tag == "RightHand" )
         {
             LeftHolsterTouching = true;
         }
       
     }
     public void OnTriggerExit(Collider other)
     {
         if (other.tag == "LeftHand" || other.tag == "SwordL" || other.tag == "RightHand")
         {
             LeftHolsterTouching = false;
         }
     }
 
     public void LeftHolsterHandL()
     {
         if (LeftHolsterTouching)
         {
             if (!LeftSwordTaken && !settings.LeftSwordEnabled && !InRightHand)
             {
                 LeftSwordTaken = true;
                 settings.LeftSwordEnabled = true;
                 settings.LeftHandEnabled = false;
 
                 GetComponent<MeshRenderer>().enabled = false;
                 SwordL.SetActive(true);
                 HandL.SetActive(false);
 
                 InLeftHand = true;
                 InRightHand = false;
 
             } 
             else if (LeftSwordTaken && !InRightHand)
             {
                 LeftSwordTaken = false;
                 settings.LeftSwordEnabled = false;
                 settings.LeftHandEnabled = true;
 
                 GetComponent<MeshRenderer>().enabled = true;
                 SwordL.SetActive(false);
                 HandL.SetActive(true);
 
                 InLeftHand = false;
                 InRightHand = false;
 
             }
         }
     }
     public void RightHolsterHandL()
     {
         if (LeftHolsterTouching)
         {
             if (!LeftSwordTaken && !settings.RightSwordEnabled && !InLeftHand)
             {
                 LeftSwordTaken = true;
                 settings.RightSwordEnabled = true;
 
                 settings.RightHandEnabled = false;
                 GetComponent<MeshRenderer>().enabled = false;
                 SwordR.SetActive(true);
                 HandR.SetActive(false);
 
                 InLeftHand = false;
                 InRightHand = true;
             }
             else if (LeftSwordTaken && !InLeftHand)
             {
                 LeftSwordTaken = false;
                 settings.RightSwordEnabled = false;
 
                 settings.RightHandEnabled = true;
                 GetComponent<MeshRenderer>().enabled = true;
                 SwordR.SetActive(false);
                 HandR.SetActive(true);
 
                 InLeftHand = false;
                 InRightHand = false;
             }
         }
     }