GameObject becomes 'unselectable'

I have a basic infantry unit and I'm working on selecting and moving it. I have this strange problem where if I click on the unit over and over and again it selects with no problem (confirmed using a Debug.Log) but if I move the mouse slightly in any direction while still clicking on the unit the Raycast suddenly just goes through the unit and then at is will hit the plane's collider.

Here's the script for the left click select: else if(Input.GetMouseButtonUp(0) && didUserClickLeftMouse(hit.point)) {

 if(hit.collider.name == TagName.ground_name)
 {    Debug.Log("First hit: "+ hit.collider.name);
 if(!shiftKeysDown())

deselectGameObjectIfSelected(); }

 else if(hit.collider.tag == TagName.gangster_tag)
 {
 
 Debug.Log("FOUND GANG: " + hit.collider.name);
                     if(!unitAlreadySelected(hit.collider.gameObject))//If unit is not already selected
 {
                         if(shiftKeysDown())
 {
                             currentUnit = hit.collider.gameObject;
                             //currentUnit.transform.Find(TagName.unit_projector).gameObject.SetActive(true);
                             currentSelectUnits.Add(currentUnit);
 }
 else //shift keys are not pressed
                         {
                         deselectGameObjectIfSelected();
                             currentUnit = hit.collider.gameObject;
                             Debug.Log("current: " + currentUnit.name);
     
                             currentUnit.transform.Find(TagName.select_display).gameObject.SetActive(true);
                             
                             currentSelectUnits.Add(currentUnit);
                             Debug.Log("selected count: "+ currentSelectUnits.Count);
     }
 }
 else//if unit is already in the list
 {
                         if(shiftKeysDown())
                         {
                             removeUnitFromSelected(hit.collider.gameObject);
                         }
 else
 {
                             deselectGameObjectIfSelected();
                             currentUnit = hit.collider.gameObject;
                             currentUnit.transform.Find(TagName.select_display).gameObject.SetActive(true);
                             currentSelectUnits.Add(currentUnit);
 }
 }
                     
                     
 }//end of if hit collider crew
 }//end of mouse click input

 public static void deselectGameObjectIfSelected()
     {
         
         if(currentSelectUnits.Count > 0)
         {
             GameObject arrayUnit;
             for(int i = 0;i< currentSelectUnits.Count;i++)
             {
                 arrayUnit = currentSelectUnits[i] as GameObject;
                 arrayUnit.transform.Find(TagName.select_display).gameObject.SetActive(false);
             }
             currentSelectUnits.Clear();
             Debug.Log("selected count(Clear): "+ currentSelectUnits.Count);
         }
     }

 public static bool unitAlreadySelected(GameObject unit)
     {
         if(currentSelectUnits.Count > 0)
         {
             GameObject arrayUnit;
             for(int i = 0;i< currentSelectUnits.Count;i++)
             {
                 arrayUnit = currentSelectUnits[i] as GameObject;
                 if(unit == arrayUnit)
                 {
                     return true;
                 }
             }
         }
         return false;
     }

 public bool didUserClickLeftMouse(Vector3 hitpoint)
     {
         if(
             (leftDownPoint.x < hitpoint.x + clickDragZone && leftDownPoint.x > hitpoint.x - clickDragZone) &&
             (leftDownPoint.y < hitpoint.y + clickDragZone && leftDownPoint.y > hitpoint.y - clickDragZone) &&
             (leftDownPoint.z< hitpoint.z + clickDragZone && leftDownPoint.z > hitpoint.z - clickDragZone)
             )
             return true; else return false;
     }

I really have no idea on this one, it seems pretty weird that I can no longer select the unit at all. Any help anyone can offer would be greatly appreciated.