Make VR HUD For health bar and mana.

Well I figured it out. Here is the solution I used. I basically make the object act as if it were a child of the Camera only via code this way I can still manipulate the position propperly. Then Vector3.SmoothDamp and the Github I found with Quanternion SmoothDamp, take care of the position and rotation.

 using UnityEngine;
 
 public class HUDTracker : MonoBehaviour
 {
 
     private Transform fakeParent;
     private Vector3 _positionOffset;
     private Quaternion _rotationOffset;
     private Camera cameraObj;
     private Transform cameraTransform;
     private Vector3 velocity = Vector3.zero;
 
     void Start()
     {
         cameraObj = Camera.main;
         transform.localPosition = new Vector3(cameraObj.transform.localPosition.x - 0.3f, cameraObj.transform.localPosition.y - 2.2f, cameraObj.transform.localPosition.z + 1.2f); 
 
         // Set parent object
         SetFakeParent(Camera.main.transform);
 
         cameraTransform = cameraObj.transform;
     }
 
     // Update is called once per frame
     void Update()
     {
         // this is the desired position for the HUD.
         Vector3 offsetHUDPos = cameraTransform.TransformPoint(new Vector3(cameraObj.transform.localPosition.x, cameraObj.transform.localPosition.y - 2.2f, cameraObj.transform.localPosition.z + 1.2f));
         
         var temp = new Quaternion(0.3f, 0.3f, 0.3f, 0.3f);
         transform.rotation = SmoothDamp(transform.rotation, fakeParent.rotation * _rotationOffset, ref temp, 0.05f);
 
         // Change position with SmoothDamp so it moves smoothly.
         transform.position = Vector3.SmoothDamp(transform.position, offsetHUDPos, ref velocity, 0.2f);
     }
 
     public void SetFakeParent(Transform parent)
     {
         //Offset vector
         _positionOffset = transform.InverseTransformPoint(transform.position) - transform.InverseTransformPoint(parent.position);
         //Offset rotation
         _rotationOffset = Quaternion.Inverse(parent.rotation) * transform.rotation;
         //Our fake parent
         fakeParent = parent;
     }
 
     private Quaternion SmoothDamp(Quaternion rot, Quaternion target, ref Quaternion deriv, float time)
     {
         if (Time.deltaTime < Mathf.Epsilon) return rot;
         // account for double-cover
         var Dot = Quaternion.Dot(rot, target);
         var Multi = Dot > 0f ? 1f : -1f;
         target.x *= Multi;
         target.y *= Multi;
         target.z *= Multi;
         target.w *= Multi;
         // smooth damp (nlerp approx)
         var Result = new Vector4(
             Mathf.SmoothDamp(rot.x, target.x, ref deriv.x, time),
             Mathf.SmoothDamp(rot.y, target.y, ref deriv.y, time),
             Mathf.SmoothDamp(rot.z, target.z, ref deriv.z, time),
             Mathf.SmoothDamp(rot.w, target.w, ref deriv.w, time)
         ).normalized;
 
         // ensure deriv is tangent
         var derivError = Vector4.Project(new Vector4(deriv.x, deriv.y, deriv.z, deriv.w), Result);
         deriv.x -= derivError.x;
         deriv.y -= derivError.y;
         deriv.z -= derivError.z;
         deriv.w -= derivError.w;
 
         return new Quaternion(Result.x, Result.y, Result.z, Result.w);
     }
 
     /* 
      * SmoothDamp function acquired from https://gist.github.com/maxattack/4c7b4de00f5c1b95a33b
      * Copyright 2016 Max Kaufmann (max.kaufmann@gmail.com)
      * Permission is hereby granted, free of charge, to any person obtaining a copy
      * of this software and associated documentation files (the "Software"), to deal
      * in the Software without restriction, including without limitation the rights
      * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
      * copies of the Software, and to permit persons to whom the Software is
      * furnished to do so, subject to the following conditions:
      *
     */
 }