Rotate the same way no matter what direction facing?

Hello!

Im having some trouble trying to get the halls to rotate properly. I'm trying to set up the controls in such a way that when you right click any wall, it rotates the level down 90 degrees and left clicks rotate the level up 90 degrees(using the FPC as the pivot point); no matter which direction the walls are facing. Right now it only works correctly on some of the walls (it's reversed on the opposite wall)and I'm not sure why.

 var Player : Transform;
 var rotateAmount = 90;
 var controller : CharacterController;
 var Hit : RaycastHit;
 
 private var isRotating = false;
 
 function FixedUpdate () {    
     //casts a ray from camera to point in world
    var ray = Camera.main.ScreenPointToRay (Input.mousePosition);
     // If the controller is grounded and the ray hits an object; then you can Rotate walls
     if (controller.isGrounded && Physics.Raycast(ray, Hit, 50)&&(isRotating==false)) {
     
         var Hitobject : GameObject = Hit.transform.gameObject;
         var pivot : Transform = Hitobject.transform;
         Debug.DrawLine(transform.position, Hit.point);
             canRotate = true;  
     
             if((Input.GetMouseButtonDown(0))&&(Hitobject.gameObject.tag=="Wall")){
                 RotateObjectF(Player.position,pivot.right, 90,1);                            
                 }     
             if((Input.GetMouseButtonDown(1))&&(Hitobject.gameObject.tag=="Wall")){
                 RotateObjectF(Player.position,-pivot.right,90,1);                            
                 }     
 }
 }
 function RotateObjectF(point : Vector3, axis : Vector3, rotateAmount : float, rotateTime : float) {
     var step : float = 0.0; //non-smoothed
     var rate : float = 1.0/rotateTime; //amount to increase non-smooth step by
     var smoothStep : float = 0.0; //smooth step this time
     var lastStep : float = 0.0; //smooth step last time
 
     while(step < 1.0) { // until we're done
         isRotating = true;
         Debug.Log("Rotating");
         step += Time.deltaTime * rate; //increase the step
         smoothStep = Mathf.SmoothStep(0.0, 1.0, step); //get the smooth step
         transform.RotateAround(point, axis, 
                                rotateAmount * (smoothStep - lastStep));
         lastStep = smoothStep; //store the smooth step
         yield;
     }
     isRotating = false;
     if (isRotating == false){    
     Debug.Log("not Rotating");
     }
     //finish any left-over
     if(step > 1.0) transform.RotateAround(point, axis,
                                           rotateAmount * (1.0 - lastStep));
 }

The left side is how the script works now and the right is what I'm trying to achieve.

Any help would be greatly appreciated :)