How do I get triggers to work at a pixel perfect level?

I'm trying to make a simple top down game in 3D where a ball bounces around. Currently, I have a square for the ball to bounce in, but the onTriggerEnter() is firing when the ball is nearly halfway through the wall before bouncing; see attachment for more detail.

Why is this happening, and is there a way to prevent it? The reason it is an issue for me is that the bouncing logic I'm using is getting screwy when the ball drifts off of the 'perfect' diagonal lines of the grid I imagined it gliding over.

I'm setting rigid.velocity directly to manipulate the direction, which I know isn't recommended, could that be causing my problem?

Here's my code:

 void bounce(GameObject bouncee)
     {
         shotController = GameObject.FindWithTag("shot").GetComponent<ShotController>();
         speed = shotController.speed;
         currentDir = shotController.getDirection();
         
         direction = new Vector3[5] {new Vector3(0, 0, 0), /* [0] : Stop */
                                     new Vector3(speed, 0, speed), /* [1] : NE */
                                     new Vector3(-speed, 0, speed), /* [2] : SE */
                                     new Vector3(-speed, 0, -speed), /* [3] : SW */
                                     new Vector3(speed, 0, -speed)}; /* [4] : NW */
 
         Vector3 tilePosition = gameObject.transform.position;
         Vector3 shotPosition = bouncee.transform.position;
 
         double xdiff = Mathf.Round(tilePosition.x - shotPosition.x);
         double zdiff = Mathf.Round(tilePosition.z - shotPosition.z);
 
         if(xdiff == 0) // Ball in same row
         {
             if(zdiff > 0) // Ball in col to left
             {
                 if(currentDir == direction[1]) // NE
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[4]); // NW
                 }
                 else // SE
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[3]); // SW
                 }
             }
             else //(zdiff < 0) // Ball in col to right
             {
                 if (currentDir == direction[4]) // NW
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[1]); // NE
                 }
                 else // SW
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[2]); // SE
                 }
             }
         }
         else if(xdiff > 0) // Ball in row below
         {
             if(zdiff == 0) // Ball in same col
             {
                 if (currentDir == direction[1]) // NE
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[2]); // SE
                 }
                 else // NW
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[3]); // SW
                 }
             }
             else if(zdiff > 0) // Ball in col to left
             {
                 shotController.adjust();
                 shotController.setDirection(direction[0]); // Stop
             }
             else //(zdiff < 0) // Ball in col to right
             {
                 shotController.adjust();
                 shotController.setDirection(direction[0]); // Stop
             }
         }
         else //(xdiff < 0) // Ball in row above
         {
             if(zdiff == 0) // Ball in same col
             {
                 if (currentDir == direction[2]) // SE
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[1]); // NE
                 }
                 else // SW
                 {
                     shotController.adjust();
                     shotController.setDirection(direction[4]); // NW
                 }
             }
             else if(zdiff > 0) // Ball in col to left
             {
                 shotController.adjust();
                 shotController.setDirection(direction[0]); // Stop
             }
             else //(zdiff < 0) // Ball in col to right
             {
                 shotController.adjust();
                 shotController.setDirection(direction[0]); // Stop
             }
         }
            
     }

I want this to work without having to adjust() the shot's position before changing the direction of it's velocity. Thanks!