3D RPG - Player movement - He automatically finds the fastest way and goes around everything. How to limit his movement?

Hello guys, I'm working on a rpg game like Diablo. I wanted to make my character move when I Right click on the terrain and I did it by making two scripts - one for the movement itself and one for the controllers (left / right clicking) BUT he finds the fastest way to go to the place. For example if I have made something like barriers between two zones. My character is in Zone 1. When I click on Zone 2 he automatically finds the fastest way and goes around the barriers till he is in the place I clicked in Zone 2. I want him to move when I click somewhere but I don't want him to explore the map himself by finding the fastest way.

I will put my movement script here:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 using UnityEngine.AI;
 
 /* This component moves our player using a NavMeshAgent. */
 
 [RequireComponent(typeof(NavMeshAgent))]
 public class PlayerMotor : MonoBehaviour
 {
     Transform target;       // Target to follow
     NavMeshAgent agent;     // Reference to our agent
 
     // Get references
     void Start()
     {
         agent = GetComponent<NavMeshAgent>();
     }
     void Update()
     {
         if(target != null)
         {
             agent.SetDestination(target.position);
             FaceTarget();
         }
     }
 
     public void MoveToPoint(Vector3 point)
     {
         agent.SetDestination(point);
     }
 
     public void FollowTarget(Interactable newTarget)
     {
         agent.stoppingDistance = newTarget.radius * .8f;
         agent.updateRotation = false;
         target = newTarget.interactionTransform;   
     }
     public void StopFollowingTarget()
     {
         agent.stoppingDistance = 0f;
         agent.updateRotation = true;
         target = null;      
     }
 
   void FaceTarget()
     {
         Vector3 direction = (target.position - transform.position).normalized;
         Quaternion lookRotation = Quaternion.LookRotation(new Vector3(direction.x, 0f, direction.z));
         transform.rotation = Quaternion.Slerp(transform.rotation, lookRotation, Time.deltaTime * 5f);
     }
 }

and here is the one with the controllers:

 using UnityEngine.EventSystems;
 using UnityEngine;
 
 /* Controls the player. Here we choose our "focus" and where to move. */
 
 [RequireComponent(typeof(PlayerMotor))]
 public class PlayerController : MonoBehaviour
 {
     public Interactable focus;
 
     public LayerMask movementMask;  // Filter out everything not walkable
 
     Camera cam;         // Reference to our camera
     PlayerMotor motor;  // Reference to our motor
 
     // Get references
     void Start()
     {
         cam = Camera.main;
         motor = GetComponent<PlayerMotor>();
     }
 
     // Update is called once per frame
     void Update()
     {
         if (EventSystem.current.IsPointerOverGameObject())
             return;
 
         // If we press left mouse
         if (Input.GetMouseButtonDown(0))
         {
             // We create a ray
             Ray ray = cam.ScreenPointToRay(Input.mousePosition);
             RaycastHit hit;
 
             // If the ray hits
             if (Physics.Raycast(ray, out hit, 100, movementMask))
             {
                 motor.MoveToPoint(hit.point);   // Move to where we hit
                 RemoveFocus();
             }
         }
 
         // If we press right mouse
         if (Input.GetMouseButtonDown(1))
         {
             // We create a ray
             Ray ray = cam.ScreenPointToRay(Input.mousePosition);
             RaycastHit hit;
 
             // If the ray hits
             if (Physics.Raycast(ray, out hit, 100))
             {
                 Interactable interactable = hit.collider.GetComponent<Interactable>();
                 if(interactable != null)
                 {
                     SetFocus(interactable);
                 }
             }
         }
     }
     void SetFocus(Interactable newFocus)
     {
         if (newFocus != focus)
         {
             if (focus != null)
                 focus.OnDefocused();
 
             focus = newFocus;
             motor.FollowTarget(newFocus);
         }
         newFocus.OnFocused(transform);
         
     }
     void RemoveFocus()
     {
         if (focus != null)
             focus.OnDefocused();
 
         focus = null;
         motor.StopFollowingTarget();
     }
 }