Can someone explain how this collision detection work and why it fails sometimes?

I tried to adapt the 2d platformer character controller from this live session: https://www.youtube.com/watch?v=wGI2e3Dzk_w&list=PLX2vGYjWbI0SUWwVPCERK88Qw8hpjEGd8

Into a 2d top down character controller. It seemed to work but it is possible to move into colliders with some combination of keys pressed that I couldn't really find out, but it's easy to make it happen.

The thing is I don't understand how the collision detection is really working here so I don't know how to fix it. I appreciate if someone can explain how this works.

Thanks :)

This is how the player is set up:

PlayerControllerTopDown2D.cs:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 
 public class PlayerControllerTopDown2D : PhysicsObject2D
 {
     public float maxSpeed = 7;
  
     private SpriteRenderer spriteRenderer;
     private Animator animator;
 
     private bool facingUp, facingDown, facingLeft, facingRight;
 
     void Awake()
     {
         spriteRenderer = GetComponent<SpriteRenderer>();
         animator = GetComponent<Animator>();
 
         facingUp = true;
         facingDown = facingLeft = facingRight = false;
     }
 
     protected override void ComputeVelocity()
     {
         Vector2 move = Vector2.zero;
 
         move.x = Input.GetAxis("Horizontal");
         move.y = Input.GetAxis("Vertical");
  
         targetVelocity = move * maxSpeed;
 
         if (move.y > minMoveDistance && !facingUp)
         {   
             clearOthersAndSet(0);
             // sprite rotation
         }
 
         if (move.y < -minMoveDistance && !facingDown)
         {
             clearOthersAndSet(1);
             // sprite rotation
         }
 
         if (move.x < -minMoveDistance && !facingLeft)
         {
             clearOthersAndSet(2);
             // sprite rotation
         }
 
         if (move.x > minMoveDistance && !facingRight)
         {
             clearOthersAndSet(3);
             // sprite rotation
         }
 
 
     }
 
     void clearOthersAndSet(int x)
     {
         switch (x)
         {
             case 0:
                 facingUp = true;
                 facingDown = facingLeft = facingRight = false;
                 break;
             case 1:
                 facingDown = true;
                 facingUp = facingLeft = facingRight = false;
                 break;
             case 2:
                 facingLeft = true;
                 facingUp = facingDown = facingRight = false;
                 break;
             case 3:
                 facingRight = true;
                 facingUp = facingDown = facingLeft = false;
                 break;
         }
     }
 }
 

PhysicsObject2D.cs:

 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
 
 public class PhysicsObject2D : MonoBehaviour
 {
     protected Rigidbody2D rb2d;
     protected Vector2 velocity;
     protected Vector2 targetVelocity;
 
     protected ContactFilter2D contactFilter;
     protected RaycastHit2D[] hitBuffer = new RaycastHit2D[16];
     protected List<RaycastHit2D> hitBufferList = new List<RaycastHit2D>(16);
 
 
     protected const float minMoveDistance = 0.001f;
     protected const float shellRadius = 0.01f;
 
     protected bool hitSomething = false;
     
 
     void OnEnable()
     {
         rb2d = GetComponent<Rigidbody2D>();
     }
 
     void Start()
     {
         contactFilter.useTriggers = false;
         contactFilter.SetLayerMask(Physics2D.GetLayerCollisionMask(gameObject.layer));
         contactFilter.useLayerMask = true;
     }
 
     void Update()
     {
         targetVelocity = Vector2.zero;
         ComputeVelocity();
     }
 
     protected virtual void ComputeVelocity()
     {
 
     }
 
     void FixedUpdate()
     {
         if (hitSomething)
         {
             targetVelocity = -targetVelocity * 5;
             hitSomething = false;
         }
 
         velocity.x = targetVelocity.x;
         velocity.y = targetVelocity.y;
 
         Vector2 deltaPosition = velocity * Time.deltaTime;
 
         Vector2 move = Vector2.right * deltaPosition.x;
 
         Movement(move, false);
 
         move = Vector2.up * deltaPosition.y;
 
         Movement(move, true);
     }
 
     void Movement(Vector2 move, bool yMovement)
     {
         float distance = move.magnitude;
 
         if (distance > minMoveDistance)
         {
             int count = rb2d.Cast(move, contactFilter, hitBuffer, distance + shellRadius);
             
             if (count > 0)
                 hitSomething = true;
             else
                 hitSomething = false;
 
             hitBufferList.Clear();
             for (int i = 0; i < count; i++)
             {
                 hitBufferList.Add(hitBuffer[i]);
             }
                 
             for (int i = 0; i < hitBufferList.Count; i++)
             {
                 float modifiedDistance = hitBufferList[i].distance - shellRadius;
                 distance = modifiedDistance < distance ? modifiedDistance : distance;
             }
         }
 
         rb2d.position = rb2d.position + move.normalized * distance;
     }
 }