Streaming in Unity - Poor performance issue.

Hi there !

Three days ago I started to try to make my own streaming / video chat in Unity thinking it will be quick and easy ( ha-ha-ha ). Actually I have kind of "succeed". My server run the device webcam, transform the texture in arrays of bytes and send them to the client for reconstruction. The client receive it and - woohoo - I have my image !

{ }

My problem : the time taken for the entire process for one image. It take about 6-7 seconds to send it.... What a stream ! erk.

{ }

Maybe I made a big mistake in my script but I'm so "in it" since 3 days I can't figure it out. If someone could provide some help to figure why the performance are so bad - or maybe my logic is bad.

{ }

Here is the core part of the Script on the "Screen", actually a plane with a texture

 public override void OnStartServer()
 {

     //Webcam initialisation,  so only on server.
     webcamTexture = new WebCamTexture();
     serverRenderer = GetComponent<Renderer>();
     serverRenderer.material.mainTexture = webcamTexture;
     webcamTexture.Play();

     currentTexture = new Texture2D(webcamTexture.width, webcamTexture.height);

     //mean the server can send an image
     serverReadyToStream = true;

     h = currentTexture.height;
     w = currentTexture.width ;

     //the transmitter is responsible of send bytes arrays in right order
     transmitter = GetComponent<Transmitter>();

     //Some event to catch  when datas are sent
     transmitter.OnDataComepletelySent += StreamCompletelySent;
     transmitter.OnDataFragmentSent += StreamFragmentSent;


 }

 public void Start()
 {
     this.transform.position = new Vector3(-10, 2, 25);
     this.transform.rotation = Quaternion.Euler(90, 0, 180);

     //return if server, i will set up client stuff 
     if (isServer)
     {
         return;
     }

     h = 480;
     w = 640;

     transmitter = GetComponent<Transmitter>();

     //event to catch when datas are received
     transmitter.OnDataCompletelyReceived += StreamCompletelyReceived;
     transmitter.OnDataFragmentReceived += StreamFragmentReceived;

      //client plane to put the texture on
     clientScreen = GameObject.FindObjectOfType<myScreen>();
     clientRenderer = clientScreen.GetComponent<Renderer>();

 }

 public void Update()
 {
    
     if (isServer)
     {


         if (serverReadyToStream)
         {            

              //update texture
               currentTexture.SetPixels(webcamTexture.GetPixels());
             //change texture to a byte array 
             colorInByte = currentTexture.EncodeToPNG();

             serverReadyToStream = false;
             currentTransmission++;
             Debug.Log("New data transmission started with id... " + currentTransmission);
             StartCoroutine(transmitter.SendBytesToClientsRoutine(currentTransmission, colorInByte));
         }
     }

 }


 [Server]
 private void StreamCompletelySent(int transmissionId, byte[] data)
 {
     Debug.Log(" Stream COMPLETELY Sended " + transmissionId);
     serverReadyToStream = true;
 }

 [Server]
 private void StreamFragmentSent(int transmissionId, byte[] data)
 {
   //  Debug.Log(" Stream Fragment Sended " + transmissionId);
 }

 [Client]
 private void StreamCompletelyReceived ( int transmissionId, byte[] data )
 {
     Debug.Log(" Stream COMPLETELY Received " + transmissionId);

      //server don't need the stream
     if (isServer)
     {
         return;
     }

     //image reconstruction for the client
     receivedTexture = new Texture2D(w, h);
     receivedTexture.LoadImage(data);
     clientRenderer.material.mainTexture = receivedTexture;
     clientScreen.clientReadyToStream = true;

 }

 [Client]
 private void StreamFragmentReceived ( int transmissionId, byte[] data )
 {
 }

And there it is the transmitter code, same as a script publish in an old post about sending big data.

public class Transmitter : NetworkBehaviour {

 private static readonly string LOG_PREFIX = "[" + typeof(Transmitter).Name + "]: ";
 public const int RELIABLE_SEQUENCED_CHANNEL = 0;
 private static int defaultBufferSize = 1024; //max ethernet MTU is ~1400

 private class TransmissionData
 {
     public int curDataIndex; //current position in the array of data already received.
     public byte[] data;

     public TransmissionData(byte[] _data)
     {
         curDataIndex = 0;
         data = _data;
     }
 }

 //list of transmissions currently going on. a transmission id is used to uniquely identify to which transmission a received byte[] belongs to.
 List<int> serverTransmissionIds = new List<int>();

 //maps the transmission id to the data being received.
 Dictionary<int, TransmissionData> clientTransmissionData = new Dictionary<int, TransmissionData>();

 //callbacks which are invoked on the respective events. int = transmissionId. byte[] = data sent or received.
 public event UnityAction<int, byte[]> OnDataComepletelySent;
 public event UnityAction<int, byte[]> OnDataFragmentSent;
 public event UnityAction<int, byte[]> OnDataFragmentReceived;
 public event UnityAction<int, byte[]> OnDataCompletelyReceived;

 [Server]
 public void SendBytesToClients(int transmissionId, byte[] data)
 {
     Debug.Assert(!serverTransmissionIds.Contains(transmissionId));
     StartCoroutine(SendBytesToClientsRoutine(transmissionId, data));
 }

 [Server]
 public IEnumerator SendBytesToClientsRoutine(int transmissionId, byte[] data)
 {


     Debug.Assert(!serverTransmissionIds.Contains(transmissionId));
     Debug.Log(LOG_PREFIX + "SendBytesToClients processId=" + transmissionId + " | datasize=" + data.Length);

     //tell client that he is going to receive some data and tell him how much it will be.
     RpcPrepareToReceiveBytes(transmissionId, data.Length);
     yield return null;

     //begin transmission of data. send chunks of 'bufferSize' until completely transmitted.
     serverTransmissionIds.Add(transmissionId);
     TransmissionData dataToTransmit = new TransmissionData(data);
     int bufferSize = defaultBufferSize;
     while (dataToTransmit.curDataIndex < dataToTransmit.data.Length - 1)
     {
         //determine the remaining amount of bytes, still need to be sent.
         int remaining = dataToTransmit.data.Length - dataToTransmit.curDataIndex;
         if (remaining < bufferSize)
             bufferSize = remaining;

         //prepare the chunk of data which will be sent in this iteration
         byte[] buffer = new byte[bufferSize];
         System.Array.Copy(dataToTransmit.data, dataToTransmit.curDataIndex, buffer, 0, bufferSize);

         //send the chunk
         RpcReceiveBytes(transmissionId, buffer);
         dataToTransmit.curDataIndex += bufferSize;

         yield return null;

         if (null != OnDataFragmentSent)
             OnDataFragmentSent.Invoke(transmissionId, buffer);
     }

     //transmission complete.
     serverTransmissionIds.Remove(transmissionId);

     if (null != OnDataComepletelySent)
         OnDataComepletelySent.Invoke(transmissionId, dataToTransmit.data);
 }

 [ClientRpc]
 private void RpcPrepareToReceiveBytes(int transmissionId, int expectedSize)
 {
     if (clientTransmissionData.ContainsKey(transmissionId))
         return;

     //prepare data array which will be filled chunk by chunk by the received data
     TransmissionData receivingData = new TransmissionData(new byte[expectedSize]);
     clientTransmissionData.Add(transmissionId, receivingData);
 }

 //use reliable sequenced channel to ensure bytes are sent in correct order
 [ClientRpc(channel = RELIABLE_SEQUENCED_CHANNEL)]
 private void RpcReceiveBytes(int transmissionId, byte[] recBuffer)
 {
     //already completely received or not prepared?
     if (!clientTransmissionData.ContainsKey(transmissionId))
         return;

     //copy received data into prepared array and remember current dataposition
     TransmissionData dataToReceive = clientTransmissionData[transmissionId];
     System.Array.Copy(recBuffer, 0, dataToReceive.data, dataToReceive.curDataIndex, recBuffer.Length);
     dataToReceive.curDataIndex += recBuffer.Length;

     if (null != OnDataFragmentReceived)
         OnDataFragmentReceived(transmissionId, recBuffer);

     if (dataToReceive.curDataIndex < dataToReceive.data.Length - 1)
         //current data not completely received
         return;

     //current data completely received
     Debug.Log(LOG_PREFIX + "Completely Received Data at transmissionId=" + transmissionId);
     clientTransmissionData.Remove(transmissionId);

     if (null != OnDataCompletelyReceived)
         OnDataCompletelyReceived.Invoke(transmissionId, dataToReceive.data);
 }

Thanks for reading ! :) and thanks in advance for any clue or solution !