Problem with random room gen using raycasts!

My raycasts aren't hitting any colliders. There are four raycasts (top, left, right and bottom) that are meant to check if there is a room there and if there isn't, it can generate a room there. It doesn't seem to detect any rooms though so rooms are spawned on top of other rooms sometimes.

 public class RoomGeneration : MonoBehaviour {
 
  public int roomsGenerated; // How many rooms are generated
  public GameObject[] rooms;
  public GameObject doorTop;
  public GameObject doorBottom;
  public GameObject doorLeft;
  public GameObject doorRight;
  public GameObject rayUp;
  public GameObject rayDown;
  public GameObject rayLeft;
  public GameObject rayRight;
  int genX; // Room X coord
  int genY; // Room Y coord
  // Use this for initialization
  void Start()
  {
      genX = 0;
      genY = 0;
      Instantiate(rooms[0], new Vector2(genX, genY), Quaternion.identity);
      for (int i = 0; i <= 20; i++)
      {
          CheckAndBuild();
      }
  }
  // Update is called once per frame
  void Update()
  {
     
  }
  void CheckAndBuild()
  {
      int randomRoomDir = Random.Range(0, 4);
      int randomRoom = Random.Range(1, 6);
      RaycastHit2D hitUp = Physics2D.Raycast(rayUp.transform.position, transform.up, 3f);
      RaycastHit2D hitDown = Physics2D.Raycast(rayDown.transform.position, transform.up * -1, 3f);
      RaycastHit2D hitLeft = Physics2D.Raycast(rayLeft.transform.position, transform.right * -1, 5f);
      RaycastHit2D hitRight = Physics2D.Raycast(rayRight.transform.position, transform.right, 5f);
      Debug.DrawRay(rayUp.transform.position, transform.up * 3f, Color.red, 1000f);
      Debug.DrawRay(rayDown.transform.position, transform.up * -1 * 3f, Color.red, 1000f);
      Debug.DrawRay(rayLeft.transform.position, transform.right * -1 * 5f, Color.red, 1000f);
      Debug.DrawRay(rayRight.transform.position, transform.right * 5f, Color.red, 1000f);
      if (randomRoomDir == 0 && hitUp.collider == null) // Generate room above
      {
          Instantiate(doorTop, new Vector2(genX, genY + 3.55f), Quaternion.identity);
          genY += 10;
          Vector2 roomGenPos = new Vector2(genX, genY);
          transform.position = roomGenPos;
          Instantiate(rooms[randomRoom], roomGenPos, Quaternion.identity);
          Instantiate(doorBottom, new Vector2(genX, genY - 3.55f), Quaternion.identity);
          print(hitUp.collider);
      }
      if (randomRoomDir == 1 && hitDown.collider == null) // Generate room below
      {
          Instantiate(doorBottom, new Vector2(genX, genY - 3.55f), Quaternion.identity);
          genY -= 10;
          Vector2 roomGenPos = new Vector2(genX, genY);
          transform.position = roomGenPos;
          Instantiate(rooms[randomRoom], roomGenPos, Quaternion.identity);
          Instantiate(doorTop, new Vector2(genX, genY + 3.55f), Quaternion.identity);
          print(hitUp.collider);
      }
      if (randomRoomDir == 2 && hitLeft.collider == null) // Generate room to the left
      {
          Instantiate(doorLeft, new Vector2(genX - 7.4f, genY), Quaternion.identity);
          genX -= 20;
          Vector2 roomGenPos = new Vector2(genX, genY);
          transform.position = roomGenPos;
          Instantiate(rooms[randomRoom], roomGenPos, Quaternion.identity);
          Instantiate(doorRight, new Vector2(genX + 7.4f, genY), Quaternion.identity);
          print(hitUp.collider);
      }
      if (randomRoomDir == 3 && hitRight.collider == null) // Generate room to the right
      {
          Instantiate(doorRight, new Vector2(genX + 7.4f, genY), Quaternion.identity);
          genX += 20;
          Instantiate(rooms[randomRoom], new Vector2(genX, genY), Quaternion.identity);
          Instantiate(doorLeft, new Vector2(genX - 7.4f, genY), Quaternion.identity);
          print(hitUp.collider);
      }
  }
 }

Any help would be greatly appreciated! This is driving me insane.

In the picture you can see the drawlines that show what the rays are meant to be doing. The rays come from empty gameobjects that are children of another gameobject which is what this script is attached to. When a room is generated, this object moves to the center of the new room at the same time so the raycasts are always in the right place. The raycasts just wont detect anything they collide with.

Also, the Debug.DrawLine's don't all fire properly. You can't see it in the picture but sometime they don't appear.