How can I change the button's color back to its previous state?

I'm making a Simon Says game where the system comes up with a randomly generated pattern and the player must mimic it to win. I'm having one issue, I can't make the buttons return to white after they have been changed to a different color. This is an issue because if they do not return to white, the pattern isn't clear. The // code in the case section makes the buttons continuously flash.

 using UnityEngine;
 using UnityEngine.UI;
 using System.Collections;
 using System.Collections.Generic;
 using System.Threading;
 
 public class Simon : MonoBehaviour 
 {
     // Button variables
     public Button redButton;
     public Button blueButton;
     public Button greenButton;
     public Button yellowButton;
 
     // Text variables (was using to see if I was actually clicking the buttons)
     public Text lossText;
     public Text scoreText;
 
     // Had to make onInList static so it would hold its value throughout the game
     static int onInList = 0;
     List<int> pattern = new List<int>();
     // Had to use System.Random to be able to use Random.Next() otherwise it uses Unity's version which doesn't work for this
     System.Random rand = new System.Random();
     bool isPlayingBack = false;
 
     // Variables to hold the Image component of the buttons
     private Image redRend;
     private Image blueRend;
     private Image greenRend;
     private Image yellowRend;
 
     void Start()
     {
         // Disables the lossText gameobject
         lossText.enabled = false;
 
         // Sets the Image components to their prespective variables
         redRend = redButton.GetComponent<Image> ();
         blueRend = blueButton.GetComponent<Image>();
         greenRend = greenButton.GetComponent<Image>();
         yellowRend = yellowButton.GetComponent<Image>();
     }
 
     void FixedUpdate()
     {
         // Updates the score text
         scoreText.text = "Score: " + onInList.ToString();
 
         // Adds random numbers to the next element of pattern
         pattern.Add(rand.Next(0, 4));
 
         // Starts the main coroutine
         StartCoroutine("PlayBack");
     }
 
     // This is the main coroutine method
     IEnumerator PlayBack()
     {
         // Sets isPlayingBack to true every time this coroutine starts
         isPlayingBack = true;
 
         // If isPlayingBack is true run the game
         if (isPlayingBack) 
         {
             // Test to see what number is in each of patterns elements
             foreach(int colors in pattern.ToArray()) 
             {
                 // If the element is one of these numbers (which is has to be)...
                 switch (colors) 
                 {
                 // each case changes the color of the button to the color it's supposed to be, then back to white...
                 // there's a bug that makes the buttons flash. I don't know why they flash, I'm still working on it.
                 case 0:
                     redRend.color = new Color (255, 0, 0);
                     var red = redRend.color;
                     yield return new WaitForSeconds (1);
 
                     //if (redRend.color.Equals(red))
                     //redRend.color = new Color (255, 255, 255);
                     break;
                 case 1:
                     blueRend.color = new Color (0, 0, 255);
                     var blue = blueRend.color;
                     yield return new WaitForSeconds (1); 
                 
 
                     //if (blueRend.color.Equals(blue))
                     //    blueRend.color = new Color (255, 255, 255);
                     break;
                 case 2:
                     greenRend.color = new Color (0, 255, 0);
                     var green = greenRend.color;
                     yield return new WaitForSeconds (1);
                     //if (greenRend.color.Equals(green))
                     //greenRend.color = new Color (255, 255, 255);
                     break;
                 case 3:
                     yellowRend.color = new Color (251, 255, 0);
                     var yellow = yellowRend;
                     yield return new WaitForSeconds (1);
                     // (yellowRend.color.Equals(yellow))
                     //yellowRend.color = new Color (255, 255, 255);
                     break;
                 }
                 // When the switch case statement is broken out of set isPlayingBack to false.
                 isPlayingBack = false;
                 // Wait for 4 seconds before breaking out of foreach loop.
                 yield return new WaitForSeconds (1);
             }
         }
     }
 
     // This method runs how the button clicks test if the player is correct or not
     public void TestCorrect(int color)
     {
         // If isPlayingBack is true return and do nothing.
         if (isPlayingBack)
             return;
 
         // If the element (onInList) for pattern is the same is the "color" parameter...
         if (pattern [onInList] == color) 
         {
             // then update the score text...
             scoreText.text = "Score: " + onInList;
             // and add 1 to onInList.
             onInList++;
         } 
         else 
         {
             // Otherwise, tell the player the game is over and what their score was.
             lossText.text = "Game Over! Final Score: " + pattern.Count.ToString();
             lossText.enabled = true;
         }
         // If the player lost (onInList is greater than or equal the the element count of pattern)...
         if (onInList >= pattern.Count)
         {
             // then start the game over
             pattern.Add(rand.Next(0, 4));
             StartCoroutine ("PlayBack");
             onInList = 0;
         }
     }
 
     public void ColorChange () { 
     } 
     // These are the click methods for the buttons. They just call TestCorrect with their 
     // mapped element number as the color parameter.
     public void RedClick()
     {
         TestCorrect (0);
     }
 
     public void BlueClick()
     {
         TestCorrect (1);
     }
 
     public void GreenClick()
     {
         TestCorrect (2);
     }
 
     public void YellowClick()
     {
         TestCorrect (3);
     }
 }