Raycast Steering Problem

I'm working on a raycast driven obstacle avoidance that will steer characters away from obstructions, but I'm having problems with making the character completely steer away from obstructions.

With the code I have below, the character is facing away from the obstacles until the feeler rays aren't touching, but now, the character faces right back towards the ray if the raycast hits again instead of staying steered away.

[**Text in Pic**]: The gray line represents the intended direction of the character before obstacle avoidance. The black line represents the direction the character should be steering towards after obstacle avoidance.

The orange line represents one of the feelers detecting a surface, and storing it's normal, which should be applied (trans.forward += hit.normal.normalized * Time.smoothedDeltaTime;)

Figure two represents what the error looks like: the character re-orients itself to face the player (which makes it face the wall again) after steering is done (black line), and repeats what's going on in figure one.

P.S: I am avoiding UnitySteer since I'm not really looking into having to use an outside framework.

I'm pretty sure the problem lies here in this snippet:

 var adjDir = ObstacleAvoidance(character.direction, new Vector3(2f, 1, 2f), character.checkObstacles);
 
 character.targetSpeed = character.maxSpeed;
 
 if (!isSteering)
               character.direction = new Vector3(character.direction.x * character.targetSpeed, character.direction.y , character.direction.z *character.targetSpeed);
           else
               character.direction = new Vector3(adjDir.x, character.direction.y, adjDir.z);
 
           trans.forward = new Vector3(character.direction.x, 0, character.direction.z).normalized * Time.deltaTime; 

Here is all the code I am using:

 Vector3 ObstacleAvoidance(Vector3 dir, Vector3 steering, bool checkObstacles)
     {
         List<Vector3> steeringRays = new List<Vector3>();
         var _holdTheJump = dir.y;
         
         bool left = false;
         bool right = false;
         bool front = false;
         Vector3 adjDirection = dir;
         
         steeringRays.Add(trans.TransformDirection(-steering.x, steering.y, steering.z)); //ray pointed slightly left 
         steeringRays.Add(trans.TransformDirection(steering.x, steering.y, steering.z)); //ray pointed slightly right 
         steeringRays.Add(trans.forward); //ray 1 is pointed straight ahead
         
         RaycastHit hit;
         
         if (checkObstacles )
         {
                 Debug.DrawRay(trans.localPosition, steeringRays[0].normalized * rayLength, Color.cyan);
                 Debug.DrawRay(trans.localPosition, steeringRays[1].normalized * rayLength, Color.cyan);
                 Debug.DrawRay(trans.localPosition, steeringRays[2].normalized * rayLength, Color.cyan);
 
                 if (Physics.Raycast(trans.position, steeringRays[0], out hit, rayLength))
                 {
                     if (hit.collider.gameObject.layer != 13 && (!front && !left))
                     {
                         isSteering = true;
                         front=false; right=false; left=true;
                         Debug.DrawLine(trans.position, hit.point, Color.red);
                         trans.forward += (hit.normal).normalized * Time.smoothDeltaTime;
                         Debug.Log("Steer Left");
                         return trans.forward;
                     }
                 }
                 else                
                 if (Physics.Raycast(trans.position, steeringRays[1], out hit, rayLength))
                 {
                     if (hit.collider.gameObject.layer != 13 && (!front && !left)) //Character layer
                     {
                         Debug.DrawLine(trans.position, hit.point, Color.red);
                         front=false; right=true; left=false;
                         isSteering = true;
                         trans.forward += (hit.normal).normalized * Time.smoothDeltaTime;
                         Debug.Log("Steer Right");
                         return trans.forward;
                     }
                 }
                 else 
                 {
                     isSteering = false;
                     left = false; right = false; front = false;
                     return adjDirection;
                 }
             return adjDirection;
         }
         return adjDirection;
     }
     
     public bool Seek(Vector3 targetPos)
     {        
         var tempDir = (targetPos - trans.position);
         var holdY = character.direction.y;
         
         character.direction = new Vector3(tempDir.x, 0, tempDir.z).normalized;
         character.direction.y = holdY;
         character.speed = character.targetSpeed;
         
         if (character.controller.isGrounded)
             character.jumping = false;
         
         //If not to close, but not to far
         if ((tempDir.magnitude > character.minDistance))   
         {
             //Check for potential pitfalls. Stop if there is a pitfall ahead
             RaycastHit hit;
             
             #region Ground Ahead!
             if (Physics.Raycast(trans.position + trans.forward, Vector3.down, out hit, 10))
             {
                 var adjDir = ObstacleAvoidance(character.direction, new Vector3(2f, 1, 2f), character.checkObstacles);
 
                 character.targetSpeed = character.maxSpeed;
                 
                 if (!isSteering)
                     character.direction = new Vector3(character.direction.x * character.targetSpeed, character.direction.y , character.direction.z *character.targetSpeed);
                 else
                     character.direction = new Vector3(adjDir.x, character.direction.y, adjDir.z);
                 
                 trans.forward = new Vector3(character.direction.x, 0, character.direction.z).normalized * Time.deltaTime; 
             }#endregion
             else
             #region There is a pit!
             {
                 character.targetSpeed = 0; 
 
                  if (!character.jumping)
                 {
                     character.Upward(10);    
                     character.jumping = true;
                 }
             }
             #endregion
 
               return false;
         }
         return true;
     }