Unity Mirror - Player prefab rotation not syncronised on client

I am trying to set the rotation of PlayerPrefab for players that join via NetworkManager, however server side client gets updated rotation as expected but client side does not for whatever reason. Any idea why is this happening?

 public class CustomNetworkManager : NetworkManager
 {
     [SerializeField] private Camera mainCamera;
     private List<GameObject> players = new List<GameObject>();
 
     public override void OnServerAddPlayer(NetworkConnectionToClient conn)
     {
         base.OnServerAddPlayer(conn);
 
         players.Add(conn.identity.gameObject);
         mainCamera.GetComponent<CameraController>().players.Add(conn.identity.transform);
 
         if (players.Count == 1)
         {
             Vector3 localScale = conn.identity.transform.localScale;
             localScale.x = -1f;
 
             conn.identity.transform.rotation = Quaternion.Euler(new Vector3(0f, 90f));
             conn.identity.transform.localScale = localScale;
         }
         else if (players.Count == 2) { conn.identity.transform.rotation = Quaternion.Euler(new Vector3(0f, -90f)); }
 
         if (players.Count < 2) { return; }
 
         players[0].GetComponent<CustomPlayerController>().opponent = players[1];
         players[1].GetComponent<CustomPlayerController>().opponent = players[0];
     }
 }

PlayerPrefab has NetworkTransform set and Sync Rotation is ticked. I tried adding NetworkTransformChild as well but that also didn't help.

Any idea is much appreciated, thank you!