Mouse orbit on a touchscreen

I've got my script working on the touchscreen and it orbits very nicely. Just one slight problem - After I've swiped or made a movement, then remove my finger from the touchscreen and replace my finger in a different spot on the screen the camera moves to that spot. If I touch generally in the same location that I removed my finger from, the camera will move ever so slightly. So basically the camera is moving wherever I touch the screen. I'd like it to not start moving until I start dragging (Just like the mouse performs on the pc).

Can anyone help me out with this? Thanks y'all.

 using UnityEngine;
 
 using System.Collections;
 
  
 
 public class CameraMove : MonoBehaviour {
 
 public Transform target;
 
 public float distance= 1.5f;
 
 public int cameraSpeed= 5;
 
  
 
 public float xSpeed= 175.0f;
 
 public float ySpeed= 75.0f;
 
 public float pinchSpeed;
 
 private float lastDist = 0;
 
 private float curDist = 0;
 
  
 
 public int yMinLimit= 10; //Lowest vertical angle in respect with the target.
 
 public int yMaxLimit= 80;
 
  
 
 public float minDistance= .5f; //Min distance of the camera from the target
 
 public float maxDistance= 1.5f;
 
  
 
 private float x= 0.0f;
 
 private float y= 0.0f;
 
 private Touch touch;
 
  
 
 void  Start (){
 
     Vector3 angles= transform.eulerAngles;
 
     x = angles.y;
 
     y = angles.x;
 
  
 
    // Make the rigid body not change rotation
 
       if (rigidbody)
 
       rigidbody.freezeRotation = true;
 
 }
 
  
 
 void  Update (){
 
     if (target && camera) {
 
  
 
    //Zooming with mouse
 
    distance += Input.GetAxis("Mouse ScrollWheel")*distance;
 
    distance = Mathf.Clamp(distance, minDistance, maxDistance);
 
  
 
   
 
     if (Input.touchCount == 1 && Input.GetTouch(0).phase == TouchPhase.Moved) 
 
     {
 
         //One finger touch does orbit
 
         touch = Input.GetTouch(0);
 
         x += touch.deltaPosition.x * xSpeed * 0.02f;
 
         y -= touch.deltaPosition.y * ySpeed * 0.02f;
 
     }
 
     if (Input.touchCount > 1 && (Input.GetTouch(0).phase == TouchPhase.Moved || Input.GetTouch(1).phase == TouchPhase.Moved)) 
 
     {
 
         //Two finger touch does pinch to zoom
 
         var touch1 = Input.GetTouch(0);
 
         var touch2 = Input.GetTouch(1);
 
         curDist = Vector2.Distance(touch1.position, touch2.position);
 
         if(curDist > lastDist)
 
         {
 
             distance += Vector2.Distance(touch1.deltaPosition, touch2.deltaPosition)*pinchSpeed/10;
 
         }else{
 
             distance -= Vector2.Distance(touch1.deltaPosition, touch2.deltaPosition)*pinchSpeed/10;
 
         }
 
  
 
         lastDist = curDist;
 
     }
 
      //Detect mouse drag;
 
    if(Input.GetMouseButton(0))   {
 
    
 
         x += Input.GetAxis("Mouse X") * xSpeed * 0.02f;
 
         y -= Input.GetAxis("Mouse Y") * ySpeed * 0.02f;       
 
       }
 
       y = ClampAngle(y, yMinLimit, yMaxLimit);
 
               
 
         Quaternion rotation= Quaternion.Euler(y, x, 0);
 
         Vector3 vTemp = new Vector3(0.0f, 0.0f, -distance);
 
         Vector3 position= rotation * vTemp + target.position;
 
          
 
    transform.position = Vector3.Lerp (transform.position, position, cameraSpeed*Time.deltaTime);
 
       transform.rotation = rotation;      
 
     }
 
 }
 
  
 
 static float  ClampAngle ( float angle ,   float min ,   float max  ){
 
    if (angle < -360)
 
       angle += 360;
 
    if (angle > 360)
 
       angle -= 360;
 
    return Mathf.Clamp (angle, min, max);
 
 }
 
  
 
 }