OnCollisionEnter2D and OnCollisionExit2D executed at the same time

Hi, I'm doing a simple 2D Side Scrolling Tower Defense Game. I'm facing this issue where both OnCollisionEnter2D and OnCollisionExit2D are called at the same time, hence make the code inside both of them cancelling out each other. Everything works well if just one of my object (with this script on it) is colliding with just another one object (which has the BoxCollider2D). But the problem starts to come when there is a second object (with this script) starts coming in (stacking) and the OnCollisionEnter2D and OnCollisionExit2D will start to both be executed at the same time.

I'm very very new to Unity as well as codings, so please do go easy on me. Any help would be much appreciated!

The full scripts are as follow :

 using System.Collections;
  using System.Collections.Generic;
  using UnityEngine;
  
  public class FriendlyKnightUnits : MonoBehaviour
  {
      // Components
      private Animator anim;
      private Rigidbody2D rb;
      private Collider2D coll;
  
      // FSM
      private enum State { walk, attack, death };
      private State state = State.walk;
  
      // Inspector Variables
      private float walkSpeed;
      [SerializeField] private float initialWalkSpeed = 1.1f;
  
      // Start Variables
      private Vector2 unitPosition;
  
      // Health variables
      public int maxHealth = 120;
      private int currentHealth;
      public HealthBar healthBar;
  
      public int attackDamage = 20;
  
      public Transform attackPoint;
      public float attackRange = 0.5f;
      public LayerMask enemyLayers;
  
      // Sound Variables
      private AudioSource soundSource;
      [SerializeField] private AudioClip swordSwing;
  
      void Start()
      {
          currentHealth = maxHealth;
          healthBar.SetMaxHealth(maxHealth);
  
          walkSpeed = initialWalkSpeed;
  
          anim = GetComponent<Animator>();
          rb = GetComponent<Rigidbody2D>();
          coll = GetComponent<Collider2D>();
          soundSource = GetComponent<AudioSource>();
      }
  
      void Update()
      {
          anim.SetInteger("state", (int)state);
  
          rb.velocity = new Vector2(walkSpeed, rb.velocity.y);
      }
  
      private void OnCollisionEnter2D(Collision2D collision)
      {
          if (collision.gameObject.CompareTag("Enemy"))
          {
              state = State.attack;
              walkSpeed = 0;
          }
  
          if (collision.gameObject.CompareTag("Player"))
          {
              Physics2D.IgnoreCollision(coll, collision.transform.GetComponent<Collider2D>());
          }
      }
  
      private void OnTriggerEnter2D(Collider2D collide)
      {
          if (collide.gameObject.name == "EnemyCastle")
          {
              state = State.attack;
              walkSpeed = 0;
          }
      }
  
      private void OnCollisionExit2D(Collision2D collision)
      {
          if (collision.gameObject.CompareTag("Enemy"))
          {
              walkSpeed = initialWalkSpeed;
              state = State.walk;
          }
      }
  
      void Attack()
      {
          // Detect enemies in range of attack
          Collider2D[] hitEnemies = Physics2D.OverlapCircleAll(attackPoint.position, attackRange, enemyLayers);
  
          // Damage them
          foreach (Collider2D enemy in hitEnemies)
          {
              if (enemy.GetComponent<EnemyKnightUnits>())
              {
                  enemy.GetComponent<EnemyKnightUnits>().TakeDamage(attackDamage);
              }
  
              if (enemy.GetComponent<EnemyMinerUnits>())
              {
                  enemy.GetComponent<EnemyMinerUnits>().TakeDamage(attackDamage);
              }
          }
      }
  
      private void OnDrawGizmosSelected()
      {
          if (attackPoint == null)
          {
              return;
          }
  
          Gizmos.color = Color.red;
          Gizmos.DrawWireSphere(attackPoint.position, attackRange);
      }
  
      public void TakeDamage(int damage)
      {
          currentHealth -= damage;
  
          // Set the health slider
          healthBar.SetHealth(currentHealth);
  
          if (currentHealth <= 0)
          {
              Die();
          }
      }
  
      void Die()
      {
          // Die animation
          anim.SetInteger("state", (int)State.death);
  
          // Disable your unit
          rb.gravityScale = 0f;
          coll.enabled = false;
          this.enabled = false;
          Destroy(this.gameObject, 3f);
      }
  
      private void SwordSwing()
      {
          soundSource.clip = swordSwing;
          soundSource.Play();
          Attack();
      }
  }

I'm starting to suspect that the issue might be caused from these lines of scripts at line 65 - 70, but I couldn't be sure.

 if (collision.gameObject.CompareTag("Player"))
          {
              Physics2D.IgnoreCollision(coll, collision.transform.GetComponent<Collider2D>());
          }

I need this part of the code to be in my script because I want my object to be able to stack on each other.