On-Screen joystick using PointerEventData takes input from mouse drags and touch drags anywhere on screen

I have a on-screen joystick which is used to move my ship, which works fine, but drag movements via touch and mouse also trigger it which ruins it. I know why it takes the input, because of the OnDrag method but i'm not sure how to fix it

 using UnityEngine;
 using UnityEngine.EventSystems;
 
 public class Joystick : MonoBehaviour, IDragHandler, IPointerUpHandler, IPointerDownHandler{  
     private Vector2 _position   = Vector2.zero;
     private Camera cam          = new Camera();
     private Vector2 inputVector = Vector2.zero;
 
     [Header("Components")]
     public RectTransform background;
     public RectTransform handle;
 
     [Header("Options")]
     public float handleLimit = 1f;
 
 
     [HideInInspector] public float Horizontal { get { return inputVector.x; } }
     [HideInInspector] public float Vertical   { get { return inputVector.y; } }
     [HideInInspector] public Vector2 position { get { return this._position; } }
 
     private void Start() {
         _position = RectTransformUtility.WorldToScreenPoint(cam, background.position);
     }
 
     public void OnDrag(PointerEventData eventData) {
         Vector3 direction       = eventData.position - _position;
         inputVector             = (direction.magnitude > background.sizeDelta.x / 2f) ? direction.normalized : direction / (background.sizeDelta.x / 2f);
         handle.anchoredPosition = (inputVector * background.sizeDelta.x / 2f) * handleLimit;
     }
 
     public void OnPointerDown(PointerEventData eventData) {
         OnDrag(eventData);
     }
 
     public void OnPointerUp(PointerEventData eventData) {
         inputVector             = Vector2.zero;
         handle.anchoredPosition = Vector2.zero;
     }
 }