Unloading Chunks not working

Hey Guys, I'm currently trying to create a chunk loader for my voxel game. I don't know the reason why it's not working. I can load the chunks, but my unload chunks function never unloads anything and sometimes duplicates the chunk! I've tried many other ways to remove the gameobject but they're not working for me!

 public void loadChunksForPlayer() {
         for (int x = 0; x < chunks.GetLength(0); x++) {
             for (int y = 0; y < chunks.GetLength(1); y++) {
                 for (int z = 0; z < chunks.GetLength(2); z++) {
                     float distance = Vector3.Distance(new Vector3(x * chunkSize, y * chunkSize, z * chunkSize), playerObj.transform.position);
                     if (distance < player.renderDistance) {
                         if(chunks[x,y,z] == null){
                             loadChunk(x,y,z);
                         }
                     }else if (distance > player.renderDistance) {
                         if(chunks[x,y,z] != null){
                             Debug.Log ("unload at x="+x+" y="+y+" z="+z+"   with distance="+distance);
                             unloadChunk(x,y,z);
                         }
                     }
                 }
             }
         }
     }
 
     public void loadChunk(int x, int y, int z) {
         Vector3 pos = new Vector3(x * chunkSize, y * chunkSize, z * chunkSize);
         GameObject newChunk = (GameObject)Instantiate(chunk, pos, new Quaternion(0,0,0,0)) as GameObject;
         //chunks [x,y,z] = new Chunk ();
         chunks [x,y,z] = newChunk.GetComponent<Chunk>() as Chunk;
         chunks [x,y,z].generatorGO = gameObject;
         chunks [x,y,z].chunkSize = chunkSize;
         chunks [x,y,z].chunkPos = pos;
     }
 
     public void unloadChunk(int x, int y, int z) {
         Destroy(chunks [x,y,z].transform.gameObject);
     }

(Please excuse my grammar mistakes, English is not my first language)

Thanks in advance!