Multiplayer player look at hit.point but other players twitch

I have followed the unity multiplayer tutorials and I wanted to make a multiplayer look at mouse point, the issue I have is every player that joins moves slightly when a player changes where he is facing. Really struggling to find what I am missing. any guidance would be greatly appreciated.

I have also tried have the camera disabled and turning it on for local player but does not change anything.

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.Networking;
 
 public class PlayerlookAt : NetworkBehaviour
 {
     public bool LookAt;
     public float LookSpeed = 1.0f;
     void Update () {
         if (isLocalPlayer)
         {
             RaycastHit hit;
             Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
             if (Physics.Raycast(ray, out hit))
             {
                 Vector3 relativePos = hit.point - transform.position;
                 relativePos.y = 0;
                 Quaternion rotation = Quaternion.LookRotation(relativePos);
                 transform.rotation = Quaternion.Slerp(transform.rotation, rotation, LookSpeed);
             }
         }
     }
 }