How to swap object again?

i have create the game in which object is swapping it's position with empty game object. problem occur when i click first time on the object it swap.,but when i click again on same object it does not swap it's position. what am i doing wrong..??please help

board.cs

 void Start () {
 
         for(int a=GridHeight;a>0;a--){
             for(int b=0;b<GridWidth;b++)
         {
                 
                 GameObject g =  Instantiate(gemprefab, new Vector3 ( b,a, 0), Quaternion.identity) as GameObject;
                 //g.AddComponent("GUIText");
                 //g.guiText.text = y.ToString();
                 g.transform.parent = gameObject.transform;
                 
                 
                 gemComponent = g.GetComponent<Gem>();
                 gems.Add(gemComponent);
                 
                 if(a==1 && b==3)
                 { 
                     gemComponent.cube.renderer.enabled = false;
                 }
 
         }
         gameObject.transform.position = new Vector3 (-2.5f, -2.5f, 0);
     }
 public void SwapGem(Gem gem1)
     {
 
         gem1start = gem1.transform.position;
         gem1end = gemComponent.transform.position;
         if (gemComponent.IsNeighbourWith(gem1))
         {
                         gem2start = gem1end;
                         gem2end = gem1start;
                         gem1.transform.position = gem2start;
                         gemComponent.transform.position = gem2end;
 
                 }
 
         for (int i = 0; i < gems.Count; i++) {
 
             if(gems[i].transform.position != gems[randomIndex].transform.position)
             {
                 print ("not");
             }else{
                 //Application.LoadLevel("level02");
                 print ("yes");
             }
 
                 }
     }

Feeler.cs

  void OnTriggerEnter(Collider c)
         {
             //owner.RemoveNeighbor(c.GetComponent<Gem>());
                             if (c.tag == "Gem") {
             
                                     owner.AddNeighbor(c.GetComponent<Gem> ());
                             
                     }
     
         }
         void OnTriggerExit(Collider c)
         {
             if(c.tag =="Gem")
             {
                 owner.RemoveNeighbor(c.GetComponent<Gem>());
     
     
             }
     
     
   }

Gems.cs

  public void AddNeighbor(Gem g)
         {    
                     if (!Neighbors.Contains (g)) {
                             Neighbors.Add (g);
             } 
             }
     
         public void OnMouseDown()
         {
     
                     if (!GameObject.Find ("board").GetComponent<board> ().isSwapping) {
                             GameObject.Find ("board").GetComponent<board> ().SwapGem (this);
     
                     }
                     
         }
     
             public void RemoveNeighbor(Gem g)
             {
         
             Neighbors.Remove(g);
     
             }