how to stop child object from twitching?

I'm building a character select screen that swipe. I have one main gameobject that read swiping of finger. Each character has a parent gameobject nested under this one gameobject that reads when the character is swiped out of a certain range. and I have another child gameobject that is the image of the character. So there are 3 tiers of gameobjects. When the second tier of gameobject is within 1 and -1 of world space it scales up character image and sets it to x= 0. when out of range next character is scale and x = 0. all my characters sit perfectly at zero with out any twitching, but one character does twitch. I have been trying to figure out what I did wrong. What doesn't make sense to me is all the other characters work great except this one. here is my code that handles the character scaling and setting to x = 0.

  public class ScaleCharacter : MonoBehaviour 
  {
 public Vector3         minScale;
 public Vector3        maxScale;
 public float        lerpTime;
 public GameObject    name;
 public GameObject    lockedTextGO;
 public GameObject    juggDescription;
 public GameObject    demonDescription;

 Transform            character;

 // Use this for initialization
 void Start () 
 {
     //Grabs the image of character
     foreach(Transform child in transform)
     {
         character = child;
     }
 }
 
 // Update is called once per frame
 void Update () 
 {
     if(transform.position.x < 0.999f && transform.position.x > -0.999f)
     {
         //set characters name active
         name.SetActive (true);
         //initializes the characters pos
         Vector3 characterPos = character.position;
         //Scales character up
         character.localScale = Vector3.Lerp (maxScale, minScale, 0);
         //Sets character to middle of the screen
         characterPos = new Vector3(0f, -1.0f, -5f);
         //Sets characters pos
         character.position = characterPos;
         
     }
     else{
         /*
          * What to do when the character isn't in the middle of the screen
          */ 
         name.SetActive (false);
         character.localScale = Vector3.Lerp (minScale, maxScale, 0);
         character.localPosition = Vector3.zero;
     }
 }

}