Can't Add OnClick.Add.Listener To Multiplayer UI

I can't add listener to player's UI button.Somehow I can get button name with:

  GameObject.Find("Button").GetComponent<Button>().name

But,adding listeners always fail.

Other elements like text fields are working flawlessly.

UI elements are editor added so I'm not instantiating them. Local player authority is set.I can use CmdPressButton methode with:

 if (Input.GetKey(KeyCode.Space))

Here is the only script I'm using.

 using UnityEngine;
 using System.Collections;
 using UnityEngine.Networking;
 using UnityEngine.UI;
 
 public class Player : NetworkBehaviour {
     public int score;
     public Text scoreText;
     
     void Start()
     {
         if(isServer)
         {
             GameObject.Find("Button").GetComponent<Button>().onClick.RemoveAllListeners();
             GameObject.Find("Button").GetComponent<Button>().onClick.AddListener(CmdPressButton);
         }
         else
         {
             if (isLocalPlayer)
             {
                 scoreText = GameObject.Find("Score").GetComponent<Text>();
                 scoreText.text = "Zero.";
                 GameObject.Find("Button").GetComponent<Button>().onClick.RemoveAllListeners();
                 GameObject.Find("Button").GetComponent<Button>().onClick.AddListener(CmdPressButton);               
             }
         }        
     }
 
 
     //Server Side.
     [Server]
     void UpdateScore()
     {
         score++;
         OnScoreChanged();
     }
 
     [Server]
     void OnScoreChanged()
     {
         RpcUpdateLocalScore(score);
     }
 
     [Command]
     void CmdPressButton()
     {
         score += 123;
         OnScoreChanged();
     }
 
     //Client Side.   
     [ClientRpc]
     void RpcUpdateLocalScore(int newScore)
     {
         if (isLocalPlayer && !isServer)
         {
             scoreText.text = newScore.ToString();
         }
     }
 }