Rigidbody insists on adding torque around child object's origin [C#]

So I'm simply trying to Add torque to an empty game object that has a cube as a child. No matter what I do, the Toque is applied as if It was adding toque to the cube. I want to add toque based off of the parent object's (the object that has the script attached) origin. It's adding it based off of the child cube's origin. Here's my code.

 void FixedUpdate () {
         h = Input.GetAxis("Horizontal");
 
         Debug.Log (h);
 
         if (h > 0) {
             rigidbody.AddRelativeTorque(0,speed,0);    
         }
 
         if (h < 0) {
             rigidbody.AddRelativeTorque(0,-speed,0);    
         }
 
         if (h == 0) {
             rigidbody.AddTorque(Vector3.up * 0);    
         }
 
         if (Input.GetKeyDown ("i")) {
             Inside.gameObject.SetActive(true);
             Outside.gameObject.SetActive(false);
         }
         if (Input.GetKeyDown ("o")) {
             Inside.gameObject.SetActive(false);
             Outside.gameObject.SetActive(true);
         }
     }
 }