Problem With Raycast Pickup Using Touch

So I have this script attached to my Character Controller to pickup objects with the "Pickupable" script attached to them.

 using UnityEngine;
 using System.Collections;
 
 public class PickupObject : MonoBehaviour {
     GameObject mainCamera;
     bool carrying;
     GameObject carriedObject;
     public float distance;
     public float smooth;
     // Use this for initialization
     void Start () {
         mainCamera = GameObject.FindWithTag("MainCamera");
     }
     
     // Update is called once per frame
     void Update () {
         if(carrying) {
             carry(carriedObject);
             checkDrop();
             //rotateObject();
         } else {
             pickup();
         }
     }
 
     void rotateObject() {
         carriedObject.transform.Rotate(5,10,15);
     }
 
     void carry(GameObject o) {
         o.transform.position = Vector3.Lerp (o.transform.position, mainCamera.transform.position + mainCamera.transform.forward * distance, Time.deltaTime * smooth);
     }
 
     void pickup() {
         if(Input.GetMouseButtonDown(0)) {
             Ray ray = Camera.main.ScreenPointToRay (Input.GetTouch(0).position);
             RaycastHit hit;
             if(Physics.Raycast(ray, out hit)) {
                 Pickupable p = hit.collider.GetComponent<Pickupable>();
                 if(p != null) {
                     carrying = true;
                     carriedObject = p.gameObject;
                     p.gameObject.rigidbody.isKinematic = true;
                 }
             }
         }
     }
 
     void checkDrop() {
         if(Input.GetMouseButtonDown(0)) {
             dropObject();
         }
     }
 
     void dropObject() {
         carrying = false;
         carriedObject.gameObject.rigidbody.isKinematic = false;
         carriedObject = null;
     }
 }
 

I'm using Unity's built-in first person touch controller to navigate.

I can pick up the object properly as intended as I just touch on the object in front and it immediately starts hovering in front of me just like how I want, but the problem comes when I try to touch any of the navigation pads as the cube just drops all of a sudden. I want to make it so that the cube only gets dropped when I specifically touch it. How do I do that?