How would I find neighbours of a tile in a grid ?

Hello everyone, I am currently trying to create a small builder game.

So I have arrived at a massive problem.

SOLUTION CAN BE FOUND HERE !!! https://github.com/FilipStudeny/Cellular-Automata---Island-generation

I can generate a grid that is made out of cubes each cube has its own script (Tile.cs) and that script holds references to its four neighbours (Above,Right,Left,Bellow), however I have a problem figuring out how to add these neighbours to the tiles.

I don't want to use Raycast or Sphere for neighbour detection as it slows down performance.

This is my code for the grid generation.

 public class WorldGenerator : MonoBehaviour
 {
     public int mapSize;
 
     public GameObject tile;
     public List<GameObject> tiles;
 
     // Start is called before the first frame update
     void Start()
     {
         GenerateStartingGrid();
         AssignTileNeighbors();
     }
 
     void GenerateStartingGrid() 
     {
         for (int x = 0; x < mapSize; x++)
         {
             for (int y = 0; y < mapSize; y++)
             {
                 if ( mapFilled[x,y] == 0) 
                 {
                     Vector3 tilePosition = new Vector3(-mapSize / 2 + x, 0, -mapSize / 2 + y);
 
                     GameObject newTile = Instantiate(tile, tilePosition, Quaternion.Euler(Vector3.right * 90f)) as GameObject;   
                     newTile.transform.SetParent(transform, false);
                     
                     tiles.Add(newTile);        
                 }
                 else { continue; }
                 
             }
         }
     }
 
     void AssignTileNeighbors() 
     {
         for (int x = 0; x < mapSize; x++)
         {
             for (int y = 0; y < mapSize; y++)
             {
                //Left neighbr
                 if (x - 1 >= 0)
                 {
                    
                 }
                 //Right neighbor
                 if (x + 1 < mapSize)
                 {
                    
                 }
                 //Below neighbor
                 if (y - 1 >= 0)
                 {
                     
                 }
                 //Above neighbor
                 if (y + 1 < mapSize)
                 {
                    
                 }
             }
         }
     }
 }

and this is my code for the tile.

 public class Tile : MonoBehaviour
 {
     public int ground;
 
     public GameObject neighbor_UP;
     public GameObject neighbor_RIGHT;
     public GameObject neighbor_LEFT;
     public GameObject neighbor_DOWN;
 }