all Rigid body's slow down and speed up

You could either do some homework on rigidbody characters, or just use a CharacterController. Unity installer has an asset pack you can import, makes things easy, unless you're wanting to do something different because you want to learn. If that's the case, here's something to get you started. This exmaple is written in C-Sharp.

 //RigidbodyCharacterController.cs
 
 
 using UnityEngine;
 using System.Collections;
  
 public class RigidbodyCharacterController: MonoBehaviour {
  
         public int turnSpeed;
         public int moveSpeed;
         public int jumpForce;
        
         bool onGround;
        
         // Use this for initialization
         void Start () {
        
         }
        
         // Update is called once per frame
         void FixedUpdate () {
        
                 float h = Input.GetAxis("Horizontal");
                 float v = Input.GetAxis("Vertical");
                
                 transform.Rotate( 0, h * turnSpeed * Time.deltaTime, 0 );
                 Vector3 moveAmount = transform.forward * v * moveSpeed;
                 rigidbody.MovePosition( transform.position + moveAmount * Time.deltaTime );
                 rigidbody.velocity = moveAmount + Vector3.Scale(rigidbody.velocity, new Vector3(0,1,0));       
                
                 if (onGround && Input.GetKey(KeyCode.Space))
                 {
                         rigidbody.AddForce( transform.up * jumpForce, ForceMode.Impulse );     
                         onGround = false;
                 }
         }
        
         void OnCollisionEnter() {
        
                 onGround = true;
                
         }
        
 }