Networked 3rd person aiming

Hello! I am working on a multiplayer TPS and can't seem to figure out an issue I'm having with aiming online.

I'm sending a ray from the viewpoint forward and getting the distance between the weapon and the hitinfo to create a float and using that float to determine how far the weapon should point. It looks like this-

 using UnityEngine;
 using System.Collections;
 
 public class UseAbilities : MonoBehaviour {
 
     private PlayerCharacter playerChar;
 
 
     public GameObject bullet;
 
     private Transform myTransform;
 
     private Transform leftHandTransform;
 
     private Transform rightHandTransform;
 
     private Vector3 launchPos = new Vector3();
 
     private float fireRate = 0.1f;
 
     private float nextFire;
 
     private Quaternion launchRot = new Quaternion();
 
     private Vector3 crossHairVec = new Vector3();
     private RaycastHit rayHit;
     private float distance;
 
     private RaycastHit raycastHit;
 
     private bool hit;
 
     // Use this for initialization
     void Start () {
 
         playerChar = transform.GetComponent<PlayerCharacter>();
 
         myTransform = transform;
 
         leftHandTransform = playerChar.weaponMountL.transform;
 
         rightHandTransform = playerChar.weaponMountR.transform;
 
 
     }
     
     // Update is called once per frame
     void Update () {
 
         if(Physics.Raycast(Camera.main.ViewportPointToRay(new Vector3(0.5f,0.5f,0)),out rayHit,1000f)){
             if(rayHit.transform.tag == "ProjectileObstruction" /*||rayHit.transform.tag == "Player"*/){
                 distance = Vector3.Distance(Camera.main.transform.position, rayHit.transform.position);
             }
         }
         
         if(distance == null){
             distance = 1000f;
         }
         
         if(distance < 1){
             distance = 1000f;
         }
 
         crossHairVec = Camera.main.ViewportToWorldPoint(new Vector3(0.5f,0.5f,distance));
 
         if(Input.GetButton("Fire1") && Time.time > nextFire && Network.peerType == NetworkPeerType.Disconnected && Screen.lockCursor == true){
             
             launchPos = leftHandTransform.TransformPoint(0,-0.1f,0);
 
             launchRot = Quaternion.LookRotation(crossHairVec);
 
             nextFire = Time.time + fireRate;
 
             Instantiate(bullet,launchPos, launchRot);
 
 
         }
         if(Input.GetButton("Fire1") && Time.time > nextFire && Network.peerType != NetworkPeerType.Disconnected && Screen.lockCursor == true){
 
             launchPos = leftHandTransform.TransformPoint(0,-0.1f,0);
 
             launchRot = Quaternion.LookRotation(crossHairVec);
             
             nextFire = Time.time + fireRate;
             
             networkView.RPC("ProjectileRPC",RPCMode.All,launchPos,launchRot);
             
         }
 
     }
 
     [RPC]
     void ProjectileRPC(Vector3 lp, Quaternion lr){
         Instantiate(bullet,lp,lr);
         }
 }

It seems to work offline, but online it shoots at very odd angles, almost as if it's using the server cam to aim but I know that's not the case. The camera can zoom in and out by the way, which is one reason I'm doing it like this. Any ideas?

P.S. Ignore the fact player tag is commented out, just doing some testing to make sure it wasn't hitting something on my model.