How do I keep the player camera pointed at the same spot while I rotate the player?

I have made a gravity controller script that points the player's gravity "down" based on the face normal of the object you point at. Upon switching, the player will lerp to point toward the new "up" and fall toward the new "down". The problem I am having is that the player camera rotates with the player when lerping. preferably, I would like the camera to continue to point toward the area I am lerping toward instead of turning 90 or 180 degrees WITH the player body.

I have attached a gif showing the process of flipping the gravity 180 degrees. As you can hopefully see, since the player camera is pointing down toward the floor, when the player flips, the camera is now facing the ceiling, making the flip disorientating. How could I ensure my camera continues to point towards the area where It was pointing just before I started the gravity switch?

Thank you for any suggestions, here is my code:

public class gravityController : MonoBehaviour {

 private Rigidbody rb;

 public Vector3 gravityUp;

 Vector3 localUp;

 bool gravCastHasHit = false;

 private void Start()
 {

     rb = GetComponent<Rigidbody>();

     localUp = transform.up;

     gravityUp = new Vector3(0, 9.8f, 0);

     gravCastHasHit = false;


 }


 private void Update()
 {
     if (Input.GetMouseButtonDown(0))
     {

         RaycastObjects();

     }

     if (gravCastHasHit == true)
     {

 //face player toward the current gravity "downward"

         lerpToGravDown();
     }

 



 }
 void FixedUpdate()
 {
     doCurrentGravity();

 }

 IEnumerator hitCastReset()
 {
 
     yield return new WaitForSeconds(1);    //Wait one frame
   
     gravCastHasHit = false;
  
 }

 void doCurrentGravity()
 {
     //add force to player toward current "down"

     rb.AddForce((-gravityUp * 3) * rb.mass);

 }


 void lerpToGravDown()
 {
    

     //lerp player from previous "downward" to new "downward"
     
     Vector3 LerpDir = Vector3.Slerp(transform.up, gravityUp, 5f * Time.deltaTime);
     transform.rotation = Quaternion.FromToRotation(transform.up, LerpDir) * transform.rotation;
  
 }


 void RaycastObjects()
 {
     RaycastHit hit;

     Camera cam = GetComponentInChildren<Camera>();

   
     Debug.Log("fireing ray!");
     Ray ray = cam.ScreenPointToRay(Input.mousePosition);


     if (Physics.Raycast(ray, out hit))
     {
         //Transform objectHit = hit.transform;
         localUp = transform.up;
         gravityUp = hit.normal.normalized*9.8f;
         gravCastHasHit = true;

         StartCoroutine(hitCastReset());
     }
 }

}

And here is my look script if that helps: (Note: the camera is directly parented to the player object.)

public class PlayerLook : MonoBehaviour {

 [SerializeField] private float sensX = 100f;
 [SerializeField] private float sensY = 100f;

 [SerializeField] Transform cam = null;

 float mouseX;
 float mouseY;

 float multiplier = 0.01f;

 float xRotation;
 float yRotation;

 public float mouseSensitivity = 400f;

 private void Start()
 {
     Cursor.lockState = CursorLockMode.Locked;
     Cursor.visible = false;
     gravCastHit = gravCtrl.gravCastHasHit;
     lookdir = gravCtrl.lookPoint;

 }

 private void Update()
 {

         float mouseX = Input.GetAxisRaw("Mouse X") * mouseSensitivity * Time.deltaTime;

         float mouseY = Input.GetAxisRaw("Mouse Y") * mouseSensitivity * Time.deltaTime;

         xRotation -= mouseY;
         yRotation += mouseX;

         xRotation = Mathf.Clamp(xRotation, -90f, 90f);

         cam.localRotation = Quaternion.Euler(xRotation, yRotation, 0f);
 }

}