Custom Hashset inspector Issues

I have managed to implement custom inspector for HashSet. However, there are two main issues I need help with.

1) Changes made to Hashset on a prefab/object with Itemslot.cs work but will only last till I quit Unity. In addition, when I make change to prefab and the drag it to scene, newly created object has its HashSet without any changes I made previously to prefab.

2) Since Implemented this, editor sometimes freezes indefinetely when I make changes to .cs files and it tries to compile.

How it looks:

Itemslot.cs

 public class Itemslot : MonoBehaviour, IDropHandler, IPointerEnterHandler, IPointerExitHandler
 {
 [System.Serializable]
 public class HashSetItemType : HashSet<GlobalEnums.ItemType> { }
 public HashSetItemType typeAllowed = new HashSetItemType();

 [System.Serializable]
 public class HashSetClassType : HashSet<GlobalEnums.GameClasses> { }
 public HashSetClassType classAllowed = new HashSetClassType();

 public int inventoryIndex;
 public GameObject socket;

 }

ItemslotEditor.cs

 [CustomEditor(typeof(Itemslot))]
 public class ItemslotEditor : Editor
 {

 
 public override void OnInspectorGUI()
 {        
     serializedObject.Update();
     Itemslot component = (Itemslot)target;

     GUILayout.BeginVertical("box");

     EditorGUILayout.PropertyField(serializedObject.FindProperty("socket"));
     EditorGUILayout.PropertyField(serializedObject.FindProperty("inventoryIndex"));

     var style = new GUIStyle(GUI.skin.button);
     int counter = 0;


     EditorGUILayout.LabelField("Allowed Types");

     GUILayout.BeginHorizontal();

     foreach (GlobalEnums.ItemType t in Enum.GetValues(typeof(GlobalEnums.ItemType)))
     {
         if (counter % 3 == 0 && counter > 1)
         {
             GUILayout.EndHorizontal();
             GUILayout.BeginHorizontal();
         }

         if (component.typeAllowed.Contains(t))
         {
             GUI.backgroundColor = Color.green;
         }
         else
         {
             GUI.backgroundColor = Color.white;
         }

         if (GUILayout.Button(t.ToString(), style, GUILayout.Width(120), GUILayout.Height(30)))
         {
             if (component.typeAllowed.Contains(t))
             {
                 component.typeAllowed.Remove(t);
             }
             else
             {
                 component.typeAllowed.Add(t);
             }
         }

         counter++;

     }

     GUILayout.EndHorizontal();
     GUILayout.BeginHorizontal();

     GUI.backgroundColor = Color.red;
     if (GUILayout.Button("NONE", style, GUILayout.Width(150), GUILayout.Height(25)))
     {
         component.typeAllowed.Clear();
     }

     GUI.backgroundColor = Color.yellow;
     if (GUILayout.Button("ALL", style, GUILayout.Width(150), GUILayout.Height(25)))
     {
         foreach (GlobalEnums.ItemType t in Enum.GetValues(typeof(GlobalEnums.ItemType)))
         {
             component.typeAllowed.Add(t);
         }
     }

     GUILayout.EndHorizontal();

     EditorGUILayout.LabelField("Allowed Classes");
     counter = 0;

     GUILayout.BeginHorizontal();

     foreach (GlobalEnums.GameClasses t in Enum.GetValues(typeof(GlobalEnums.GameClasses)))
     {
         if (counter % 3 == 0 && counter > 1)
         {
             GUILayout.EndHorizontal();
             GUILayout.BeginHorizontal();
         }

         if (component.classAllowed.Contains(t))
         {
             GUI.backgroundColor = Color.green;
         }
         else
         {
             GUI.backgroundColor = Color.white;
         }

         if (GUILayout.Button(t.ToString(), style, GUILayout.Width(120), GUILayout.Height(30)))
         {
             if (component.classAllowed.Contains(t))
             {
                 component.classAllowed.Remove(t);
             }
             else
             {
                 component.classAllowed.Add(t);
             }
         }

         counter++;

     }

     GUILayout.EndHorizontal();
     GUILayout.BeginHorizontal();

     GUI.backgroundColor = Color.red;
     if (GUILayout.Button("NONE", style, GUILayout.Width(150), GUILayout.Height(25)))
     {
         component.classAllowed.Clear();
     }

     GUI.backgroundColor = Color.yellow;
     if (GUILayout.Button("ALL", style, GUILayout.Width(150), GUILayout.Height(25)))
     {
         foreach (GlobalEnums.GameClasses t in Enum.GetValues(typeof(GlobalEnums.GameClasses)))
         {
             component.classAllowed.Add(t);
         }
     }

     GUILayout.EndHorizontal();
     GUILayout.EndVertical();

     serializedObject.ApplyModifiedProperties();

 }
 
 }