Unity OVR Throwing Issue.

So, for the past couple of days, I've been trying to add a velocity from the GetLocalControllerVelocity() method in the OculusSDK to a cube in order to Throw it. It works fine when turning around in the physical world, and throws in the right direction. However when I turn, the Player artificially, it doesn't seem to work as intended. I've searched and searched for answers but to no avail. Attached is a video of what i am speaking about along with my code;

 public class GrabV2 : MonoBehaviour {
 
     public float grabRadius;
 
     public LayerMask grabMask;
 
     GameObject grabbedObject;
     bool grabbing;
 
     public OVRInput.Controller controller;
 
     // Use this for initialization
     void Start () {
         
     }
     
     // Update is called once per frame
     void FixedUpdate () {
         if(OVRInput.Get(OVRInput.Button.SecondaryHandTrigger))
         {
             //print("ihkgjyhfg");
             GrabObject();
         }
 
         if (grabbing == true && !OVRInput.Get(OVRInput.Button.SecondaryHandTrigger))
         {
             //print("ihkgjyhfg");
             DropObject();
         }
 
         print(OVRInput.GetLocalControllerVelocity(controller));
 
     }
 
     void GrabObject()
     {
         grabbing = true;
 
         RaycastHit[] hits;
 
         hits = Physics.SphereCastAll(transform.position, grabRadius, transform.forward, 0f, grabMask);
 
         if(hits.Length > 0 )
         {
             int closestHit = 0;
 
             for(int i = 0; i < hits.Length; i++)
             {
                 if(hits[i].distance < hits[closestHit].distance)
                 {
                     closestHit = i;
                 }
             }
 
             grabbedObject = hits[closestHit].transform.gameObject;
 
             grabbedObject.GetComponent<Rigidbody>().isKinematic = true;
             grabbedObject.transform.position = transform.position;
             grabbedObject.transform.SetParent(transform);
         }
     }
 
     void DropObject()
     {
         grabbing = false;
 
         if(grabbedObject != null)
         {
             grabbedObject.transform.SetParent(null);
             grabbedObject.GetComponent<Rigidbody>().isKinematic = false;
 
             
             Vector3 linearVelocity = OVRInput.GetLocalControllerVelocity(controller);
             Vector3 angularVelocity = OVRInput.GetLocalControllerAngularVelocity(controller);
 
             grabbedObject.transform.GetComponent<Rigidbody>().velocity = linearVelocity;
 
             grabbedObject = null;
 
 
         }
     }
 }

Any Help would be greatly appreciated!!! Thank you