Third Person Multi-player Swapping Controls

Hello, I'm making an MMORPG, and I am having a problem. I got a server start, client connecting to server, and spawning, but I can't seem to figure out what's going on with the controller. When another client enters the server, the host and client swap. So if I had the host's player active, it would move the client, and the client's player active it would move the host's player. Please help. Here is the code:

Player.cs:

 using UnityEngine;
 using System.Collections;
 
 public class Player : MonoBehaviour
 {
     public float speed = 10f;
     public float rSpeed = 300f;
 
     private float lastSynchronizationTime = 0f;
     private float syncDelay = 0f;
     private float syncTime = 0f;
     public Vector3 syncStartPosition = Vector3.zero;
     private Vector3 syncEndPosition = Vector3.zero;
 
     void OnSerializeNetworkView(BitStream stream, NetworkMessageInfo info)
     {
         Vector3 syncPosition = Vector3.zero;
         Vector3 syncVelocity = Vector3.zero;
         if (stream.isWriting)
         {
             syncPosition = rigidbody.position;
             stream.Serialize(ref syncPosition);
 
             syncPosition = rigidbody.velocity;
             stream.Serialize(ref syncVelocity);
         }
         else
         {
             stream.Serialize(ref syncPosition);
             stream.Serialize(ref syncVelocity);
 
             syncTime = 0f;
             syncDelay = Time.time - lastSynchronizationTime;
             lastSynchronizationTime = Time.time;
 
             syncEndPosition = syncPosition + syncVelocity * syncDelay;
             syncStartPosition = rigidbody.position;
         }
     }
 
     void Awake()
     {
         lastSynchronizationTime = Time.time;
     }
 
     void Update()
     {
         if (networkView.isMine)
         {
             InputMovement();
             InputColorChange();
         }
         else
         {
             SyncedMovement();
         }
     }
 
 
     private void InputMovement()
     {
         if (Input.GetKey(KeyCode.W))
             transform.Translate(Vector3.forward * Time.deltaTime * speed);
 
         if (Input.GetKey(KeyCode.S))
             transform.Translate(Vector3.back * Time.deltaTime * speed);
 
         if (Input.GetKey(KeyCode.D))
             transform.Rotate(Vector3.up * Time.deltaTime * rSpeed);
 
         if (Input.GetKey(KeyCode.A))
             transform.Rotate(Vector3.down * Time.deltaTime * rSpeed);
     }
 
     private void SyncedMovement()
     {
         syncTime += Time.deltaTime;
 
         rigidbody.position = Vector3.Lerp(syncStartPosition, syncEndPosition, syncTime / syncDelay);
     }
 
 
     private void InputColorChange()
     {
         if (Input.GetKeyDown(KeyCode.R))
             ChangeColorTo(new Vector3(Random.Range(0f, 1f), Random.Range(0f, 1f), Random.Range(0f, 1f)));
     }
 
     [RPC] void ChangeColorTo(Vector3 color)
     {
         renderer.material.color = new Color(color.x, color.y, color.z, 1f);
 
         if (networkView.isMine)
             networkView.RPC("ChangeColorTo", RPCMode.OthersBuffered, color);
     }
 }

NetworkManager.cs:

 using UnityEngine;
 using System.Collections;
 
 public class NetworkManager : MonoBehaviour
 {
     private const string typeName = "Very Unique For the world 1111";
     private const string gameName = "Toontown Unity Test Server";
 
     private bool isRefreshingHostList = false;
     private HostData[] hostList;
 
     public GameObject playerPrefab;
 
     void OnGUI()
     {
         if (!Network.isClient && !Network.isServer)
         {
             if (GUI.Button(new Rect(100, 100, 250, 100), "Start Server"))
                 StartServer();
 
             if (GUI.Button(new Rect(100, 250, 250, 100), "Refresh Hosts"))
                 RefreshHostList();
 
             if (hostList != null)
             {
                 for (int i = 0; i < hostList.Length; i++)
                 {
                     if (GUI.Button(new Rect(400, 100 + (110 * i), 300, 100), hostList[i].gameName))
                         JoinServer(hostList[i]);
                 }
             }
         }
     }
 
     private void StartServer()
     {
         Network.InitializeServer(5, 25000, !Network.HavePublicAddress());
         MasterServer.RegisterHost(typeName, gameName);
     }
 
     void OnServerInitialized()
     {
         SpawnPlayer();
     }
 
 
     void Update()
     {
         if (isRefreshingHostList && MasterServer.PollHostList().Length > 0)
         {
             isRefreshingHostList = false;
             hostList = MasterServer.PollHostList();
         }
     }
 
     private void RefreshHostList()
     {
         if (!isRefreshingHostList)
         {
             isRefreshingHostList = true;
             MasterServer.RequestHostList(typeName);
         }
     }
 
 
     private void JoinServer(HostData hostData)
     {
         Network.Connect(hostData);
     }
 
     void OnConnectedToServer()
     {
         SpawnPlayer();
     }
 
 
     private void SpawnPlayer()
     {
         Network.Instantiate(playerPrefab, Vector3.up * 5, Quaternion.identity, 0);
     }
 }