Network Rigidbodies act weird?

Hello, so I have some players in my game (player controller script using rigidbodies) I'm using Photon Networking and everything is working but, when the two players collide in the game, the collision is very weird, it like moves... but not how it should. Think of a ball going North at 30 mph (ball a). Now think of another ball coming from the West at 60 mph(ball b). ball b hits ball a but all ball a does is goes Northnortheast... at 15mph... Sometimes ball a doesn't even seem like it got hit. Also, I tried throwing in a sphere, with a rigidbody and no network stuff, and the collisions were perrfffffect. If anyone could help me with this problem it would be great. Thanks. Here is my Character Controller:

 using UnityEngine;
 using System.Collections;
 
 public class BallMoverP2 : Photon.MonoBehaviour {
     
     
     public bool IsGrounded;
     
     
     public Camera firstPerson;
     public Camera thirdPerson;
     public float moveSpeed;
     public float jumpSpeed;
     public bool CamToggler;
 
 
     
     void Start ()
     {
         //if (Camera == null)
         //Camera = Camera.FindGameObjectsWithTag ("Camera");
         CamToggler = true;
     }
 
 
     void FixedUpdate(){
 
             if (Input.GetButtonDown ("ToggleCamera")) {
                 //CamToggler = ! CamToggler;
             }
         
             if (Input.GetButton ("Left") && CamToggler == true) {
             GetComponent<Rigidbody> ().AddForce (-firstPerson.transform.right * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody> ().AddTorque (-firstPerson.transform.right * moveSpeed * Time.deltaTime);
             }
             if (Input.GetButton ("Left") && CamToggler == false) {
             GetComponent<Rigidbody> ().AddForce (-thirdPerson.transform.right * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody>().AddTorque( -thirdPerson.transform.right * moveSpeed * Time.deltaTime);
             }
 
         
             if (Input.GetButton ("Right") && CamToggler == true) {
             GetComponent<Rigidbody> ().AddForce (firstPerson.transform.right * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody> ().AddTorque (firstPerson.transform.right * moveSpeed * Time.deltaTime);
             }
             if (Input.GetButton ("Right") && CamToggler == false) {
             GetComponent<Rigidbody> ().AddForce (thirdPerson.transform.right * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody>().AddTorque( thirdPerson.transform.right * moveSpeed * Time.deltaTime);
             }
         
             if (Input.GetButton ("Forward") && CamToggler == true) {
             GetComponent<Rigidbody> ().AddForce (firstPerson.transform.forward * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody> ().AddTorque (firstPerson.transform.forward * moveSpeed * Time.deltaTime);
             }
             if (Input.GetButton ("Forward") && CamToggler == false) { 
             GetComponent<Rigidbody> ().AddForce (thirdPerson.transform.forward * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody>().AddTorque( thirdPerson.transform.forward * moveSpeed * Time.deltaTime);
             }
         
             if (Input.GetButton ("Backward") && CamToggler == true) {
             GetComponent<Rigidbody> ().AddForce (-firstPerson.transform.forward * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody> ().AddTorque (-firstPerson.transform.forward * moveSpeed * Time.deltaTime);
             }
             if (Input.GetButton ("Backward") && CamToggler == false) {
             GetComponent<Rigidbody> ().AddForce (-thirdPerson.transform.forward * moveSpeed * Time.deltaTime, ForceMode.Impulse);
                 //GetComponent<Rigidbody>().AddTorque( -thirdPerson.transform.forward * moveSpeed * Time.deltaTime);
             }
             if (IsGrounded == true && Input.GetButtonDown ("Jump")) {
                 IsGrounded = false;
                 GetComponent<Rigidbody> ().AddForce (Vector3.up * jumpSpeed, ForceMode.Impulse);
                 //GetComponent<Rigidbody> ().AddTorque (Vector3.up * jumpSpeed, ForceMode.VelocityChange);
             }
             if (Input.GetButton ("Run")) {
                 moveSpeed = 200.0f;
             } else {
                 moveSpeed = 100.0f;
             }
 
         
         
     }
 
     
 
     void OnCollisionEnter(Collision collisionInfo)
     {
         IsGrounded = true;
     }
     
     //void OnCollisionExit (Collision collisionInfo)
     //{
     //    IsGrounded = false;
     //}
 }

And my NetworkCharacterRead/writer:

 using UnityEngine;
 using System.Collections;
 
 public class NetworkCharacter : Photon.MonoBehaviour {
     Vector3 realPosition = Vector3.zero;
     Quaternion realRotation = Quaternion.identity;
     Vector3 realVelocity = Vector3.zero;
 
     // Use this for initialization
     void Start () {
     
     }
     
     // Update is called once per frame
     void FixedUpdate () {
     if (photonView.isMine) {
             //do nothing
         } else {
             GetComponent<Rigidbody>().position = Vector3.Lerp(GetComponent<Rigidbody>().position, realPosition, 0.1f);
             GetComponent<Rigidbody>().rotation = Quaternion.Lerp(GetComponent<Rigidbody>().rotation, realRotation, 0.1f);
             GetComponent<Rigidbody>().velocity = Vector3.Lerp(GetComponent<Rigidbody>().velocity, realVelocity, 0.1f);
         }
 
     }
     public void OnPhotonSerializeView(PhotonStream stream, PhotonMessageInfo info)
     {
 
         if (stream.isWriting){
         stream.SendNext (GetComponent<Rigidbody>().position);
         stream.SendNext (GetComponent<Rigidbody>().rotation);
         stream.SendNext (GetComponent<Rigidbody>().velocity);
     }else{
             realPosition = (Vector3)stream.ReceiveNext();
             realRotation = (Quaternion)stream.ReceiveNext();
             realVelocity = (Vector3)stream.ReceiveNext();
 
 
         }
     }
 }

And my Network Manager:

 using UnityEngine;
 using System.Collections;
 
 public class NetworkManager : Photon.MonoBehaviour {
 
     public GameObject StandbyCamera;
     SpawnSpot[] spawnSpots;
     public bool CamToggler;
 
     // Use this for initialization
     void Start () {
         spawnSpots = GameObject.FindObjectsOfType<SpawnSpot> ();
         Connect ();
 
     }
 
     
     void Connect()
     {
         PhotonNetwork.ConnectUsingSettings("v1.0");
         Debug.Log ("Connecting...Baby");
     }
     void OnGUI() 
     {
         GUILayout.Label (PhotonNetwork.connectionStateDetailed.ToString ());
     }
 
     void OnJoinedLobby()
     {
         Debug.Log ("JoinedLobbyBaby");
         PhotonNetwork.JoinRandomRoom ();
     }
     void OnPhotonRandomJoinFailed()
     {
         Debug.Log("CRAP!JoinRandomLobbyFailed");
         PhotonNetwork.CreateRoom (null);
     }
     void OnJoinedRoom()
     {
         Debug.Log ("JoinedRoomBaby");
 
         SpawnMyPlayer ();
     }
     void SpawnMyPlayer ()
     {
         if (spawnSpots == null) {
             Debug.LogError ("CRAP! There is no freaking spawn spots!?");
             return;
         }
         SpawnSpot mySpawnSpot = spawnSpots [Random.Range (0, spawnSpots.Length)];
         Debug.Log ("LET THERE BE A PERSON");
         GameObject myPlayerGo = (GameObject)PhotonNetwork.Instantiate ("PlayerController", mySpawnSpot.transform.position, mySpawnSpot.transform.rotation, 0);
             
         StandbyCamera.SetActive (false);
         myPlayerGo.GetComponent <BallMoverP2> ().enabled = true;
         //myPlayerGo.GetComponent <MouseLook> ().enabled = true;
         GameObject myCamGo = myPlayerGo.transform.Find ("AXLEP1").gameObject;
         GameObject CamGo1st = myCamGo.transform.Find ("Cam1st").gameObject;
         GameObject CamGo3rd = myCamGo.transform.Find ("Cam3rd").gameObject;
         CamGo3rd.SetActive (false);
         CamGo1st.SetActive (true);
 
     }
 

Thanks for taking the time to read this far :)