[System.Serializable] Custom class use itself as variable

No, you can't do like that, please note that unity's serialization system doesn't support null, so it's simply get instantianted. Unity doesn't want to serialize your code since it leads to infinite cycle of serializations.

it's kinds of silly but to make it works, you need to manage your own serialization system with ISerializationCallbackReceiver

 //Implement ISerializationCallbackReceiver to get 
 //OnBeforeSerialize and OnAfterDeserialize working
 public class VoxelRoot : MonoBehaviour, ISerializationCallbackReceiver
 {
     //Voxel class that is used at runtime
     public class Voxel
     {
         public Vector3 vectorData = Vector3.forward;
         public List<Voxel> directions = new List<Voxel>();
     }
 
     //Voxel class that we will use for serialization
     [Serializable]
     public struct SerializableVoxel
     {
         //Insert All Variables Here
         public Vector3 vectorData;
         //Voxels Length
         public int childCount;
         //The index of first child (in serializedVoxels)
         public int indexOfFirstChild;
     }
 
     //the root of what we use at runtime. not serialized.
     Voxel root = new Voxel(); 
 
     //the field we give unity to serialize.
     public List<SerializableVoxel> serializedVoxels;
 
     public void OnBeforeSerialize()
     {
         //unity is about to read the serializedVoxels field's contents. lets make sure
         //we write out the correct data into that field "just in time".
         serializedVoxels.Clear();
         AddVoxelToSerializedVoxels(root);
     }
 
     void AddVoxelToSerializedVoxels(Voxel n)
     {
         var serializedVoxel = new SerializableVoxel () {
             //Insert All Variables Here
             vectorData = n.vectorData,
             //Additional Info for SerializableVoxel
             childCount = n.directions.Count,
             indexOfFirstChild = serializedVoxels.Count+1
         };
         serializedVoxels.Add (serializedVoxel);
         foreach (var child in n.directions)
             AddVoxelToSerializedVoxels (child);
     }
 
     public void OnAfterDeserialize()
     {
         //Unity has just written new data into the serializedVoxels field.
         //let's populate our actual runtime data with those new values.
 
         if (serializedVoxels.Count > 0)
             root = ReadVoxelFromSerializedVoxels (0);
         else
             root = new Voxel ();
     }
 
     Voxel ReadVoxelFromSerializedVoxels(int index)
     {
         var serializedVoxel = serializedVoxels [index];
         var directions = new List<Voxel> ();
         for(int i=0; i!= serializedVoxel.childCount; i++)
             directions.Add(ReadVoxelFromSerializedVoxels(serializedVoxel.indexOfFirstChild + i));
     
         return new Voxel() {
             //Insert All Variables Here
             vectorData = serializedVoxel.vectorData,
             directions = directions
         };
     }
 }

so the point is make those nested classes to just single list of classes at serialization runtime, or in other words, simply wrap everythings to single list of voxel classes before serialization, and after serialization, refill the list back to our nested classes again.

PS : The code is modified version from here with additional informations, take a look of that link if you want to more understand about serialization