Player Movement using lerp

Do you want that max distance still? If not just turn of the lerping while you are moving. If you do, just move the Lerping to Update. The that jittering is caused by desync between the movement in update and the movement in fixed update.

I made you a little "AI" code, that uses a "Coroutine" to move back. Switch off all your codes, only add mine to your player, and assign the "Start Position && "Speed in the inspector. I never worked in Unity3D, but tried to convert it to "3D" world space. :)

Maybe this is too advanced for what you want? :D

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 
 public class playerController : MonoBehaviour
 {
     public float speed;
     private float modifiedSpeed; //the speed we will modify to move back
     public Transform startPosition; //assign in inspector -> this is most likely your chair
     bool stopUserInput; // a bool to prevent any other input, and wait till the user returns
 
     bool horizontalMove; //used in the co-routine to move you
     bool veriticalMove;  //used in the co-routine to move you
 
     private Vector3 positionFromStart; //used in the co-routine to determine your orientation
     private Vector3 move;
 
     void Start()
     {
         transform.position = startPosition.position;
     }
 
     // Update is called once per frame
     void Update()
     {
         float x = Input.GetAxis("Horizontal");
         float z = Input.GetAxis("Vertical");
 
         move = transform.right * x + transform.forward * z;
 
         if (!stopUserInput) //If we are allowed to move
         {
             //If we are not at our start pos yet
             if (move == new Vector3(0,0,0) && transform.position != startPosition.position) 
             {
                 stopUserInput = true;
                 StartCoroutine(walkBackToStartPos());
             }
             else
             {
                 transform.Translate(move * speed * Time.deltaTime); //else move normally
             }
         }
 
         #region Movement for Co-Routine
         if (horizontalMove) //move with the set speed in the co-routine (walkBackToStartPos)
         {
             transform.Translate(transform.right * modifiedSpeed * Time.deltaTime);
         }
         if (veriticalMove)
         {
             transform.Translate(transform.forward * modifiedSpeed * Time.deltaTime);
         }
         #endregion
     }
 
 
     IEnumerator walkBackToStartPos()
     {
         if (transform.position.z < startPosition.position.z) //move up
         {
 
             modifiedSpeed = speed;
             veriticalMove = true;
             yield return new WaitUntil(() => transform.position.z >= startPosition.position.z);
             veriticalMove = false;
             if (transform.position.x < startPosition.position.x) //moveRight
             {
                 modifiedSpeed = speed;
                 horizontalMove = true;
                 yield return new WaitUntil(() => transform.position.x >= startPosition.position.x);
                 horizontalMove = false;
                 transform.position = startPosition.position;
                 print("Moved Back");
                 stopUserInput = false;
             }
             else //moveLeft
             {
                 modifiedSpeed = -speed;
                 horizontalMove = true;
                 yield return new WaitUntil(() => transform.position.x <= startPosition.position.x);
                 horizontalMove = false;
                 transform.position = startPosition.position;
                 print("Moved Back");
                 stopUserInput = false;
             }
 
         }
         else //moveDown
         {
             modifiedSpeed = -speed;
             veriticalMove = true;
             yield return new WaitUntil(() => transform.position.z <= startPosition.position.z);
             veriticalMove = false;
             if (transform.position.x < startPosition.position.x) //moveRight
             {
                 modifiedSpeed = speed;
                 horizontalMove = true;
                 yield return new WaitUntil(() => transform.position.x >= startPosition.position.x);
                 horizontalMove = false;
                 transform.position = startPosition.position;
                 print("Moved Back");
                 stopUserInput = false;
             }
             else //moveLeft
             {
                 modifiedSpeed = -speed;
                 horizontalMove = true;
                 yield return new WaitUntil(() => transform.position.x <= startPosition.position.x);
                 horizontalMove = false;
                 transform.position = startPosition.position;
                 print("Moved Back");
                 stopUserInput = false;
             }
         }
 
     }
 
 }

I did something else, I just placed some cubes in order to prevent me from going to far. Thank you for your support!