Faux Gravity, spinning at bottom of sphere

I'm working on a faux style gravity simulation (like mario galaxy) I have it working about 90% but when the player is running around the sphere, after a few rotations they become locked into spinning at the bottom of the sphere I'm completely out of ideas on how to deal with this so any input would be greatly helpful

</p> <pre><code>public float MoveSpeed = 20.0f; public float TurnSpeed = 10.0f; private Vector3 playerGravity = new Vector3(0, -3, 0); private Animation ani; private CharacterController controller; private float Angle = 0.0f; private Quaternion quat = Quaternion.identity; // Use this for initialization void Start () { ani = transform.FindChild("body").animation; controller = transform.GetComponent("CharacterController") as CharacterController; } void UpdateFloorRotation() { GameObject floor = GameObject.FindGameObjectWithTag("Level"); Vector3 floorVec = -transform.position.normalized; playerGravity = floorVec * 0.1f; RaycastHit hitInfo; if (Physics.Raycast(transform.position, floorVec, out hitInfo, 10, 1 &lt;&lt; 8)) { Debug.Log("Floor: " + hitInfo.collider.name); transform.rotation = quat * Quaternion.FromToRotation (Vector3.up, transform.position.normalized); Debug.Log("Normal: " + floorVec); } } // Update is called once per frame void Update () { Vector3 Locomotion = Vector3.zero; Vector2 gravity = Physics.gravity; Physics.gravity = playerGravity; if (Input.GetButtonDown("Jump")) { if (ani != null) { //ani.CrossFade("use"); } } if (Input.GetButtonUp("Up") || Input.GetButtonUp("Down") || Input.GetButtonUp("Left") || Input.GetButtonUp("Right")) { if (ani != null) { ani.CrossFade("idle"); } } if (Input.GetButton("Up")) { Locomotion += (-transform.forward * MoveSpeed); if (ani != null) { ani.CrossFade("run"); } } if (Input.GetButton("Down")) { Locomotion += (transform.forward * MoveSpeed); if (ani != null) { ani["walk"].speed = -1; ani.CrossFade("walk"); } } if (Input.GetButton("Left")) { Angle += -TurnSpeed * 1; if (ani != null &amp;&amp; !Input.GetButton("Up")) { ani["walk"].speed = 1; ani.CrossFade("walk"); } } if (Input.GetButton("Right")) { Angle += TurnSpeed * 1; if (ani != null &amp;&amp; !Input.GetButton("Up")) { ani["walk"].speed = 1; ani.CrossFade("walk"); } } Locomotion *= Time.deltaTime; quat = Quaternion.AngleAxis(Angle, playerGravity.normalized); Vector3 targetDir = transform.TransformDirection(Vector3.down); if (!Physics.Raycast(transform.position, targetDir, 3.3f, 1 &lt;&lt; 8)) { Debug.Log("Falling!"); Locomotion += playerGravity; } controller.Move(Locomotion); UpdateFloorRotation(); } </code></pre> <p>