How do I use this current Quaternion to make GameObject Joints move Correctly?

I am currently using the XBox 360 Kinect and I have a GameObject that I've attached this script to:

void LateUpdate() {

      updated = -1;
     for (int i = 0; i < KinectSkeleton.LimitMaxSkeletons; ++i)
     {
         KinectSkeletonData ksd = KinectSkeleton.GetSkeletonData((uint)i);
         if (ksd.trackingState == KinectSkeletonTrackingState.NotTracked)
             continue;

         Matrix4x4 BodyFrame_1 = SkeletonTrackerUtils.GetPlayerMatrixWaist( ref ksd );
         Matrix4x4 BodyInv_1 = BodyFrame_1.inverse;

         
         Vector3 rightShoulder   = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_SHOULDER_RIGHT]).normalized;
         Vector3 rightElbow      = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_ELBOW_RIGHT]);
         Vector3 rightWrist      = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_WRIST_RIGHT]);
         Vector3 leftShoulder    = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_SHOULDER_LEFT]).normalized;
         Vector3 leftElbow       = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_ELBOW_LEFT]);
         Vector3 leftWrist       = m_Root.transform.localToWorldMatrix * ( BodyInv_1 * ksd.skeletonPositions[(int)NUISkeleton.KinectJoint.NUI_SKELETON_POSITION_WRIST_LEFT]);


         Vector3 rightUpperArm = rightElbow - rightShoulder;
         Vector3 rightLowerArm = rightWrist - rightElbow; 
        
         Vector3 leftUpperArm = leftElbow - leftShoulder;
         Vector3 leftLowerArm = leftWrist - leftElbow;

         

         // right body frame Joint Reps

         Vector3 UpperArmR = -m_rightShoulder.m_Transform.right;
         Quaternion quatUpperArmR = Quaternion.FromToRotation(UpperArmR, rightUpperArm);
         m_rightShoulder.m_Transform.rotation = quatUpperArmR * m_rightShoulder.m_Transform.rotation;

         Vector3 lowerArmR = -m_rightElbow.m_Transform.forward;
         Quaternion quatLowerArmR = Quaternion.FromToRotation(lowerArmR, rightLowerArm);
         m_rightElbow.m_Transform.rotation = quatLowerArmR * m_rightElbow.m_Transform.rotation;

         
         // left body frame Joint Reps

         Vector3 UpperArmL = -m_leftShoulder.m_Transform.right;
         Quaternion quatUpperArmL = Quaternion.FromToRotation(UpperArmL, leftUpperArm);
         m_leftShoulder.m_Transform.rotation = quatUpperArmL * m_leftShoulder.m_Transform.rotation;

         Vector3 lowerArmL = -m_leftElbow.m_Transform.forward;
         Quaternion quatLowerArmL = Quaternion.FromToRotation(lowerArmL, leftLowerArm);
         m_leftElbow.m_Transform.rotation = quatLowerArmL * m_leftElbow.m_Transform.rotation;

         updated = i;

         KinectSkeleton.SetTrackedSkeletons(ksd.trackingId, 0);
         break;


     }

 }

This only part of the code, but I want to get the arms to move exactly as my motion of movement. Up, down, back, and forward. The problem is that my rotation is crappy when running it to the Xbox Kinect. Any suggestions on how to fix this? or how I can use the matrix4x4 to multiply the vectors correctly?