ServerReturnToLobby() not workiing

Unfortunately I don't have much more information than the title

In my LobbyManager : NetworkLobbyManager I make a call to ServerReturnToLobby(), I keep getting this error

 SetClientOwner m_ClientAuthorityOwner already set!
 UnityEngine.Networking.NetworkLobbyManager:ServerReturnToLobby()
 LobbyManager:PlayerDead(PlayerControllerComponent) (at Assets/Scripts/LobbyManager.cs:104)
 PlayerControllerComponent:CmdKillPlayer() (at Assets/Scripts/PlayerControllerComponent.cs:247)
 PlayerControllerComponent:CallCmdKillPlayer()
 PlayerControllerComponent:OnTriggerEnter2D(Collider2D) (at Assets/Scripts/PlayerControllerComponent.cs:260)

I cannot find any information about why this would be. There is nothing in the documentation, and all questions on this forum related to this issue haven't been answered yet

Here is my code, the call is being made in CheckIfGameOver() I also tried a bunch of other calls: SendReturnToLobby(), ServerReturnToLobby(), and ServerChangeScene(lobbyScene) all cause the same issue

I have a feeling I don't have something setup correctly, but I can't figure out what it is

NOTE: In order to reduce the size of this post I removed some methods that don't have anything to do with scene transitions. I can post more if needed

 public class LobbyManager : NetworkLobbyManager
 {
     private class Player
     {
         public int connectionId;
         public bool isAlive;
         public int score;
     }

     public static LobbyManager _instance;

     public RectTransform lobbyGui;
     public RectTransform menuGui;
     public RectTransform countdownGui;
     public Text countdownText;
     public float countdownTime = 5.0f;

     private List<NetworkLobbyPlayer> _players = new List<NetworkLobbyPlayer>();
     private List<Player> connectedPlayerIds = new List<Player>();         //TODO: limit number of players
     private RectTransform _currentPanel;
     private Vector3[] _playerSpawnVectors = new Vector3[4]
     {
         new Vector3(1.0f, 11.0f, 0.0f),
         new Vector3(1.0f, 1.0f, 0.0f),
         new Vector3(12.0f, 1.0f, 0.0f),
         new Vector3(12.0f, 11.0f, 0.0f)
     };

     private bool sceneLoaded = false;
     private int _gameEndCall = 0;

     public GameObject bombUpgrade;
     public GameObject laserUpgrade;
     public GameObject kickUpgrade;
     public GameObject lineUpgrade;
     public GameObject radioactiveUpgrade;

     public GameObject floor;
     public GameObject destructible;
     public GameObject indestructible;
     public GameObject[] playerAnimations;
     private BoardCreator boardCreator;

     void Start()
     {
         _instance = this;
         _currentPanel = menuGui;
     }


     // **************SERVER**************
     public override void OnLobbyStartServer()
     {
         base.OnLobbyStartServer();
         connectedPlayerIds.Clear();
     }

     public override void OnLobbyServerConnect(NetworkConnection conn)
     {
         base.OnLobbyServerConnect(conn);

         if (conn.address != "localServer")
         {
             connectedPlayerIds.Add( new Player() { connectionId = conn.connectionId, isAlive = true, score = 0 } );
         }
         
     }

     public override void OnLobbyServerDisconnect(NetworkConnection conn)
     {
         base.OnLobbyServerDisconnect(conn);
         connectedPlayerIds.Remove(connectedPlayerIds.Where(x => x.connectionId == conn.connectionId).FirstOrDefault());
     }

     public override GameObject OnLobbyServerCreateGamePlayer(NetworkConnection networkConnection, short playerControllerId)
     {
         int i = getSlotIndex(networkConnection.connectionId);

         GameObject newPlayer = (GameObject)Instantiate(playerPrefab, Vector2.zero, Quaternion.identity);
         newPlayer.transform.position = _playerSpawnVectors[i];
         newPlayer.GetComponent<PlayerControllerComponent>().playerIndex = i;

         NetworkServer.Spawn(newPlayer);
         return newPlayer;
     }

     public void PlayerDead(PlayerControllerComponent player)
     {
         SpawnUpgradeInRandomLocation(UpgradeType.Bomb, player.maxNumBombs - 1);
         SpawnUpgradeInRandomLocation(UpgradeType.Laser, player.bombParams.radius - 2);
         SpawnUpgradeInRandomLocation(UpgradeType.Kick, player.bombKick);
         SpawnUpgradeInRandomLocation(UpgradeType.Line, player.bombLine);

         connectedPlayerIds[player.playerIndex].isAlive = false;
         NetworkServer.Destroy(player.gameObject);

         CheckIfGameOver();
     }

     private void CheckIfGameOver()
     {
         if (connectedPlayerIds.Where(x => x.isAlive).Count() == 1)
         {
             connectedPlayerIds.Where(x => x.isAlive).First().score++;
             connectedPlayerIds.Where(x => x.isAlive).First().isAlive = false;
         }

         if (connectedPlayerIds.Where(x => x.isAlive).Count() == 0){
             ServerReturnToLobby(); // This call doesn't work for some reason
             Debug.Log("GAME OVER!");
         }
     }

     public override bool OnLobbyServerSceneLoadedForPlayer(GameObject gameObject1, GameObject gameObject2)
     {
         if (!sceneLoaded)
             SpawnBoard();
         sceneLoaded = true;
         return true;
     }

     public override void OnLobbyServerPlayersReady()
     {
         foreach (LobbyPlayer player in lobbySlots)
         {
             if (player != null)
             {
                 if (!player.readyToBegin)
                 {
                     return;
                 }
             }
         }

         StartCoroutine(CountDownCoroutine());
     }

     public IEnumerator CountDownCoroutine()
     {
         float remainingTime = countdownTime;
         int floorTime = Mathf.FloorToInt(remainingTime);

         while (remainingTime >= -1)
         {
             yield return null;

             remainingTime -= Time.deltaTime;
             int newFloorTime = Mathf.FloorToInt(remainingTime);

             if (newFloorTime != floorTime)
             {
                 floorTime = newFloorTime;

                 foreach (LobbyPlayer player in lobbySlots)
                 {
                     if (player != null)
                     {
                         player.RpcUpdateCountdown(floorTime);
                     }
                 }
             }
         }

         ServerChangeScene(playScene);
     }

     public void KickPlayer(NetworkConnection conn)
     {
         conn.Disconnect();
     }

     // **************CLIENT**************

     public override void OnLobbyClientEnter()
     {
         base.OnLobbyClientEnter();
         ChangePanel(lobbyGui);
     }

     private int getSlotIndex(int connectionId)
     {
         int i = 0;
         foreach (var player in connectedPlayerIds)
         {
             if (player.connectionId == connectionId)
                 return i;
             i++;
         }

         Debug.LogError("No matching playerControllerId in slots");
         throw new ArgumentOutOfRangeException();
     }


     public override void OnLobbyClientExit()
     {
         base.OnLobbyClientExit();
         ChangePanel(menuGui);
     }

     public override void OnLobbyClientSceneChanged(NetworkConnection conn)
     {
         base.OnLobbyClientSceneChanged(conn);
         _currentPanel.gameObject.SetActive(false);
     }

     // **************GUI**************

     public void ChangePanel(RectTransform newPanel)
     {
         if (_currentPanel != null)
         {
             _currentPanel.gameObject.SetActive(false);
         }

         if (newPanel != null)
         {
             newPanel.gameObject.SetActive(true);
         }

         _currentPanel = newPanel;
     }


     // **************PLAYER LIST**************

     public void AddPlayer(LobbyPlayer player)
     {
         _players.Add(player);

     }

     public void RemovePlayer(LobbyPlayer player)
     {
         _players.Remove(player);
     }
 }