Object is passing from colliders when swipe with rigidbody.position

I am new in unity and stuck on a condition where a sphere needs to collide with the walls when user swipe on screen but instead that sphere is passing through walls. here is my code.

 if (Input.touchCount == 1) 
 {
      Touch touch  = Input.GetTouch(0);
      if (touch.phase == TouchPhase.Began){
          xPos = Vector2.zero;
          xLastPos = Vector2.zero;
          xNewPos = Vector2.zero;
          xSlideMagnitude = 0;
          xPos = touch.position; 
      }
      
      else if (touch.phase == TouchPhase.Moved)
      {
          xNewPos = touch.position - xPos; 
          xLastPos = xPos;
          xPos = touch.position;
      }
      
      else if (touch.phase == TouchPhase.Stationary)
      {
          xLastPos = xPos;
          xPos = touch.position;
          xSlideMagnitude = 0.0f;
      }
      
      else if (touch.phase == TouchPhase.Ended || touch.phase == TouchPhase.Canceled)
      {
          xSlideMagnitude = 0.0f;
      }
     
     rb.position = new Vector3(gameObject.transform.position.x + xSlideMagnitude, gameObject.transform.position.y, gameObject.transform.position.z);
 }