Rigidbody2D Problems With Golf Game

I am making a mini golf game where you drag in the opposite direction you want to hit the ball and depending on how far back you drag your mouse, that's how hard you hit.

This is the code for the shooting:

 if (this.GetComponent<Rigidbody2D>().velocity.magnitude <= 0.1f && inHole == false)
         {
             ready = true;
         } else
         {
             ready = false;
             //Debug.Log(this.GetComponent<Rigidbody2D>().velocity.magnitude.ToString());
         }
 
         if (Input.GetMouseButtonDown(0) == true && !aiming && ready)
         {
             aiming = true;
         }
 
         if (targetGoal != null)
         {
             this.transform.position = Vector2.Lerp(this.transform.position, targetGoal.transform.position, 2f);
         }
 
         if (aiming)
         {
             prevPos.transform.position = this.transform.position;
             //ENABLE LINE
             line.GetComponent<LineRenderer>().enabled = true;
             //CONTROL INPUT && SHOOT POSITIONS
             startPos = this.transform.position;
             line.GetComponent<LineRenderer>().SetPosition(0, startPos);
             Vector3 shootPos = Camera.main.ScreenToWorldPoint(Input.mousePosition);
             shootPos.z = 0;
             shootPos = this.transform.position + (this.transform.position - shootPos);
             endPos = shootPos;
 
             if (Vector3.Distance(startPos, shootPos) > maxDist)
             {
                 Vector3 dir = endPos - startPos;
                 endPos = this.transform.position + (dir.normalized * maxDist);
             }
             line.GetComponent<LineRenderer>().SetPosition(1, endPos);
         } else
         {
             line.GetComponent<LineRenderer>().enabled = false;
         }
 
         if (Input.GetMouseButtonUp(0) == true && aiming && ready)
         {
             endPos = Camera.main.ScreenToWorldPoint(Input.mousePosition);
             Shoot();
         }
 
 
 
 private void Shoot()
     {
         aiming = false;
         //Debug.Log("Shooting @" + endPos.ToString());
         if(Vector2.Distance(startPos, endPos) > maxDist)
         {
 
         }
         Vector2 direction = startPos - endPos;
         this.GetComponent<Rigidbody2D>().AddForce(direction * speed * Time.deltaTime);
         score++;
         overallScore++;
         overallScoreKeeper.text = overallScore.ToString();
         scoreKeeper.text = score.ToString();
 
     }

My problem is that the speed that I shoot drastically changes to make the ball shoot forward instead of going at the speed that I set it at random times. Sometimes it goes way to fast and sometimes way to slow. I don't know why it does this.

Any help would be nice. Thank you.