Throwing dice with the mouse

I'm trying to create a pick and throw action for dice with the mouse. The idea is clicking on a die lifts the dice in the air. The die could be placed back down but a swipe and a release should throw the die and have it roll. I've been working with a 6-sided and a 20-sided. So far the lift portion works fine. I can click a die, it hovers in the air and I can release it and it drops back to the surface. What I need help with and what it isn't doing is:

Properly throwing. The throwing direction is wrong. Sometimes it looks like its throwing in the right direction and other times its throwing in the air or in an opposite direction.

I would really appreciate some help on this. I've read so many variations on this technique and tried so many things that I have completely confused myself and need to reach out for some assistance. Thanks for any help!

using UnityEngine; using System.Collections;

public class MouseDrag : MonoBehaviour {

  private Vector3 objectPosition = Vector3.zero;
 
  public enum State { None, Dragging, Throwing };
  public State status = State.None;
 
  public float h;
  public float v;
 
  public Vector3 velocity;
 
  public Vector3 lastPosition;
 
  void OnMouseDrag() {
      status |= State.Dragging;
      gameObject.layer = gameObject.layer | LayerMask.NameToLayer("Drag Layer");
 
  }
 
  void OnCollisionEnter(Collision col) {
 
  }
 
  void OnMouseUp() {
      status |= State.Throwing;
      status ^= State.Dragging;
 
      gameObject.layer = gameObject.layer ^ LayerMask.NameToLayer("Drag Layer");
  }
 
  // Use this for initialization
  void Start () {
 
  }
 
  void FixedUpdate() {
 
      h = Input.GetAxis("Mouse X") * 50f * Time.deltaTime;
      v = Input.GetAxis("Mouse Y") * 50f * Time.deltaTime;
      Rigidbody rb = GetComponent<Rigidbody>();
 
      velocity = rb.velocity;
 
      if (status == State.Dragging) {
 
          rb.AddTorque(transform.up * h, ForceMode.Impulse);
          rb.AddTorque(transform.right * v, ForceMode.Impulse);
 
          rb.useGravity = false;
          
          Ray ray =  Camera.main.ScreenPointToRay(Input.mousePosition);
          RaycastHit hit;
          
          if (Physics.Raycast(ray, out hit, 100, ~(1 << LayerMask.NameToLayer("Drag Layer")))) {
              objectPosition = new Vector3(hit.point.x, hit.point.y + .09f, hit.point.z);
              rb.MovePosition(objectPosition);
          }
 
      }
 
 
      if (status == State.Throwing) {
          status ^= State.Throwing;
 
          rb.useGravity = true;
 
          Vector3 endPos = transform.position;
          
          Vector3 force = -(endPos - lastPosition);
          //force.z = force.magnitude;
          force.Normalize();
 
          rb.AddForce(force, ForceMode.Impulse);
          rb.AddTorque(transform.up * h, ForceMode.Impulse);
          rb.AddTorque(transform.right * v, ForceMode.Impulse);
 
      }
 
      lastPosition = transform.position;
  }
  
  // Update is called once per frame
  void Update() {
 
  }

}