Unity Networking(old) Question.(Teleporting player)

Hello once again people of unity forums. I am back with yet another error. I have made a networking script and it seems like everything should work but I have a bug. The bug is that when there is more than 2 people on. The 2 clients both are stuck at the position 0,0,0. My setup is a gameObject called "MainPlayer" With two different objects as childs. One as a cube(What you see) And one is a Actual player model(What others see) here is my code:

 using UnityEngine;
 using System.Collections;
 
 public class NetworkSerialize : MonoBehaviour {
     Vector3 pos;
     Quaternion rot;
     bool wlk;
 
     void Start () {
         if (GetComponent<NetworkView> ().isMine || !Network.isClient && !Network.isServer) {
             transform.FindChild ("Player").gameObject.SetActive (false);
 
         } else {
             transform.FindChild ("FP-Player").gameObject.SetActive (false);
         }
         if(!Network.isClient && !Network.isServer){
             this.GetComponent<NetworkSerialize> ().enabled = false;
         }
     }
 
     void Update () {
         if (!GetComponent<NetworkView> ().isMine) {
             transform.FindChild("Player").localPosition=Vector3.Lerp(transform.FindChild("Player").localPosition,pos,.1f);
             transform.FindChild("Player").rotation=Quaternion.Lerp(transform.FindChild("Player").rotation,rot,.1f);
             transform.FindChild("Player").GetComponent<Animator> ().SetBool("isWalking",wlk);
         }
     }
     private void OnSerializeNetworkView(BitStream stream, NetworkMessageInfo info){
         if (stream.isWriting) {
             //if(gameObject.name.Contains("FP-Player")){
             pos=transform.FindChild("FP-Player").localPosition;
             rot=transform.FindChild("FP-Player").localRotation;
             wlk=transform.FindChild("FP-Player").GetComponent<Movement>().walking;
             stream.Serialize(ref pos);
             stream.Serialize(ref rot);
             stream.Serialize(ref wlk);
         } else {
             rot=Quaternion.identity;
             pos=Vector3.zero;
             wlk=false;
             stream.Serialize(ref pos);
             stream.Serialize(ref rot);
             stream.Serialize(ref wlk);
 
         }
     }
 }

(FP-Player is the model you see, player is the model others on the server see) I cant find anything wrong with it. Can you?