Unity Networking, Client cannot affect non-player objects

I have been struggling hard and reading a lot of answers in the forums here but nothing is working. All I am trying to get working is have two players in a game that can each pick up and toss/catch a ball. It is a Vive VR game. Every method ends with either the client cannot touch the ball at all, or can move it but the host doesn't see it move and as soon as the host touches it what the client sees is it snap back to the host.
I have started trying to use [Command] and [ClientRPC] to change and sync the position of the ball. The problem is that I get the error message that the client does not have authority to send a Command to the ball, but the AssignClientAuthority call is just not working apparently. The Command function that should be applying it doesn't even show the Debug.Log statements.

Here is the Balls Script that controls the movement. This works fine on the Host side public class BallScript : NetworkBehaviour {

 [SyncVar] public NetworkInstanceId owner;
 public Rigidbody ball;

 public void DropBall (Vector3 newVelocity, Vector3 newAngVel){
     Debug.Log ("Calling Balls DropBall");
     if (Network.isServer) {
         RpcDropBall (newVelocity, newAngVel);
     } else {
         CmdDropBall (newVelocity, newAngVel);
     }
 }

 public void MoveBall (Vector3 newPos, Quaternion newRot)
 { Debug.Log ("Calling Balls MoveBall");
     if (Network.isServer) {
         Debug.Log ("RPCMove Ball attempt"); 
         RpcMoveBall (newPos, newRot);
     } else {
         Debug.Log ("CMDMove Ball attempt"); 
         CmdMoveBall (newPos, newRot);
     }
 }

 [Command]
 void CmdMoveBall (Vector3 newPos, Quaternion newRot)
 { Debug.Log ("Calling Balls CMDMoveBall");
     if (!Network.isServer) {
         
         ball.transform.position = newPos;
         ball.transform.rotation = newRot;
     }
 }

 [Command]
 void CmdDropBall (Vector3 newVelocity, Vector3 newAngVel)
 { Debug.Log ("Calling Balls CMDDropBall");
     if (!Network.isServer) {
         ball.velocity = newVelocity;
         ball.angularVelocity = newAngVel;
         ball.useGravity = true;
     }
 }

 [ClientRpc]
 void RpcMoveBall (Vector3 newPos, Quaternion newRot ) {
     Debug.Log ("Calling Balls RPCMoveBall");
     if (Network.isServer) {
         ball.transform.position = newPos;
         ball.transform.rotation = newRot;
     }
 
 }

 [ClientRpc]
 void RpcDropBall (Vector3 newVelocity, Vector3 newAngVel) {
     Debug.Log ("Calling Balls RPCDrpBall");
     if (Network.isServer) {
         ball.velocity = newVelocity;
         ball.angularVelocity = newAngVel;
     }
 }

}

And here is the player script that handles the picking up.

public class PickupScript : NetworkBehaviour {

 public GameObject rightObject;
 public SteamVR_TrackedObject rightHand;
 public BallScript heldObject;
 public NetworkInstanceId objNetId;
 private SteamVR_Controller.Device rightController {get { return SteamVR_Controller.Input((int)rightHand.index); } } 
 public NetworkConnection conn;
 public NetworkIdentity myID;

 override public void  OnStartLocalPlayer ()
 {    base.OnStartLocalPlayer();
     objNetId = GetComponent<NetworkIdentity> ().netId;
     myID = GetComponent<NetworkIdentity> ();
      
 }

 // Update is called once per frame
 void Update () {

     if (!heldObject || !isLocalPlayer) {
         return;
     }

     Debug.Log ("HeldObject = " + heldObject.owner + "this is = " + objNetId);

     if (heldObject.owner == objNetId) {
         Debug.Log ("Calling move ball");
         heldObject.MoveBall (rightHand.transform.position, rightHand.transform.rotation);
     } else {
         heldObject = null;
     }

     if ((rightController.GetPressUp (Valve.VR.EVRButtonId.k_EButton_SteamVR_Trigger)) && (heldObject.owner == objNetId)) {
         Debug.Log ("Calling drop ball");
         heldObject.DropBall(rightController.velocity, rightController.angularVelocity);
         if (heldObject.owner == objNetId) {
             /// remove authority here, will implement later
             //heldObject.GetComponent<NetworkIdentity> ().RemoveClientAuthority (conn);
         }
         heldObject = null;
     }
 }

 void OnTriggerStay (Collider colinfo){
     if (!isLocalPlayer) {
         return;
     }
     if ((rightController.GetPressDown (Valve.VR.EVRButtonId.k_EButton_SteamVR_Trigger)) && (colinfo.tag == "Ball")) {
         Debug.Log ("Grabbing Ball");
         heldObject = colinfo.gameObject.GetComponent<BallScript>() ;
         heldObject.owner = objNetId;
         // Set Authority here!! 
         Debug.Log("calling CmdAssignowner");
         CmdAssignOwner (colinfo.GetComponent<GameObject>());
         Debug.Log ("After CmdAssignOwner call");

     }

 }
  
 [Command]
 public void CmdAssignOwner (GameObject theBall){
     Debug.Log ("Trying to assign authority");
     theBall.GetComponent<NetworkIdentity> ().AssignClientAuthority (connectionToClient);
 }

}