Script doesn't let me disable a second panel and I cant find a way around this.

Basically I have a script that controls my Pause Menu. I have a panel for the pause menu and panels for the sub menus (settings, controls etc). When I pause the game Pause panel pops up normally but if i go to settings or controls panel pops up but previous panel is still present.

This is my script. Any suggestions are welcome. Thanks.

 using UnityEngine;
 using UnityEngine.UI;
 using System.Collections;
 using UnityStandardAssets.Characters.FirstPerson;
 using UnityEngine.SceneManagement;
 
 public class PauseMenu : MonoBehaviour
 {
 
     public bool isPaused;
 
     public GameObject PausePanel;
     public GameObject quitMenu;
     public GameObject settingsMenu;
     public GameObject controlsMenu;
 
     public Button continueText;
     public Button settingsText;
     public Button controlsText;
     public Button exitText;
 
     // Use this for initialization
     void Start()
     {
  //       quitMenu = quitMenu.GetComponent<>();
  //       settingsMenu = settingsMenu.GetComponent();
  //       controlsMenu = controlsMenu.GetComponent();
 
         continueText = continueText.GetComponent<Button>();
         settingsText = settingsText.GetComponent<Button>();
         controlsText = controlsText.GetComponent<Button>();
         exitText = exitText.GetComponent<Button>();
         PausePanel.SetActive(false);
          quitMenu.SetActive(false);
         settingsMenu.SetActive(false);
         controlsMenu.SetActive(false);
 
         isPaused = false;
     }
 
     public void ContinuePress()
     {
         isPaused = false;
     }
 
     public void SettingsPress()
     {
         PausePanel.SetActive(false);
         settingsMenu.SetActive(true);
     }
 
     public void ControlsPress()
     {
         PausePanel.SetActive(false);
         controlsMenu.SetActive(true);
     }
 
     public void BackPress()
     {
         settingsMenu.SetActive(false);
         PausePanel.SetActive(true);
     }
 
     public void ExitPress() //this function will be used on our Exit button
 
     {
         quitMenu.SetActive(true); //enable the Quit menu when we click the Exit button
         continueText.enabled = false; //then disable the Play and Exit buttons so they cannot be clicked
         exitText.enabled = false;
         settingsText.enabled = false;
         controlsText.enabled = false;
     }
 
     public void NoPress() //this function will be used for our "NO" button in our Quit Menu
 
     {
         quitMenu.SetActive(false); //we'll disable the quit menu, meaning it won't be visible anymore
         continueText.enabled = true; //enable the Play and Exit buttons again so they can be clicked
         exitText.enabled = true;
         settingsText.enabled = true;
         controlsText.enabled = true;
     }
 
     public void ExitToMainMenu() //This function will be used on our "Yes" button in our Quit menu
     {
         SceneManager.LoadScene(0);//this will quit our game. Note this will only work after building the game
         isPaused = false;
     }
 
     // Update is called once per frame
     void Update()
     {
         if (isPaused)
         {
             PauseGame(true);
         }
         else {
             PauseGame(false);
         }
 
         if (Input.GetButtonDown("Cancel"))
         {
             SwitchPause();
         }
     }
 
     void PauseGame(bool state)
     {
         if (state)
         {
             PausePanel.SetActive(true);
             AudioListener.pause = true;
             Time.timeScale = 0.0f; //Paused
             GameObject.Find("Player").GetComponent<FirstPersonController>().enabled = false;
         }
         else {
             Time.timeScale = 1.0f; //Unpaused
             PausePanel.SetActive(false);
             AudioListener.pause = false;
             GameObject.Find("Player").GetComponent<FirstPersonController>().enabled = true;
         }
     }
 
     public void SwitchPause()
     {
         if (isPaused)
         {
             isPaused = false;
         }
         else {
             isPaused = true;
         }
     }
 }