Move 2 object by mouse to same position

Hi Im trying to make simple RPG where players controls 4 characters (party). Now Im struggling with that problem: Im using Raycast to point position where each character should go, but It generates problem. Characters trying to get there are stuck on trying to get to this tiny point (characters are rigidbody). The are just shaking forcing ther way to that point. How can I change that? here is my moving script

 var ray = Camera.main.ScreenPointToRay (Input.mousePosition);
         // var hit : RaycastHit;     
          //sprawdzanei czy trafilo we wroga   
          // this code show nameobject with click
          var hit : RaycastHit;
         if (Input.GetMouseButtonDown(0))
         {
 
 var hitdist = 0.0;
 
                
 
                 if (playerPlane.Raycast (ray, hitdist)) 
                 {
                     var targetPoint = ray.GetPoint(hitdist);
                     targetPosition = ray.GetPoint(hitdist);
                     var targetRotation = Quaternion.LookRotation(targetPoint - transform.position);
                     transform.rotation = targetRotation;    
 
                 }
 
              
 
             
             var dir:Vector3 = targetPosition - transform.position;
             var dist:float = dir.magnitude;
                var move:float = speed * Time.deltaTime;
             if (transform.position.y<50) transform.position.y=50;
             if(dist > move)
             {
                     transform.position += dir.normalized * move;
             } 
             else 
             {
                 transform.position = targetPosition;
         
             }
             
             transform.position += (targetPosition - transform.position).normalized * speed * Time.deltaTime; 
 
 }