Pass OpenCV coordinates to Unity

Hello!

I would like to pass x and y coordinates from OpenCV with UDP to Unity and use them in the world space. The idea is to capture movement and send the coordinates to a 3D model in Unity to move its arm. How can I adapt the coordinates that the arm is not stretched or look in any direction? I have a 400 x 400 frame. So if I get for x = 220 and for y = 180 I would like that the arm in Unity is looking in this position or transform him to it. I tested localPosition and Position in Unity and it did not worked, also ScreenToWorldPoint does not fit my final goal, does it? You can download a video where I shwo the problem: https://cloudlogin02.world4you.com/index.php/s/oQe4dyUGGKAbIiJ

I am happy for some suggestions!

Regards

Some code for OpenCV and tracking the contours:

 for c in ex_cnts:
     M = cv2.moments(c)
     (x,y,w,h) = cv2.boundingRect(c)
     if w > 20 and h > 20 and w < 100 and h < 100:
        cv2.rectangle(picture, (x,y), (x+w,y+h), (255, 0, 0), 2)
        ex_center_x = int(M['m10']/M['m00'])
        ex_center_y = int(M['m01']/M['m00'])
        if ex_center_x > center_x:
             leftx = x
             lefty = y
             print "in for " + str(y)
             messagel = "leftx " + str(leftx)
             messager = "lefty " + str(lefty)
             print messager
             # # sending data to Unity
             sock.sendto(messagel , (UDP_IP, UDP_PORT))
             sock.sendto(messager , (UDP_IP, UDP_PORT))
             left = (ex_center_x, ex_center_y)
             cv2.circle(picture, (ex_center_x, ex_center_y), 7, (255, 255, 255), -1)
             cv2.putText(picture, "left", (ex_center_x - 20, ex_center_y - 20),
                 cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
        if ex_center_x < center_x: 
             leftx = x
             lefty = y
             messagel = "leftx " + str(leftx)
             messager = "lefty " + str(lefty)
             sock.sendto(messagel , (UDP_IP, UDP_PORT))
             sock.sendto(messager , (UDP_IP, UDP_PORT))
             right = (ex_center_x, ex_center_y)
             cv2.circle(picture, (ex_center_x, ex_center_y), 7, (255, 255, 255), -1)
             cv2.putText(picture, "right", (ex_center_x - 20, ex_center_y - 20),
                 cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)

And the script I use in Unity:

 using UnityEngine;
 using System;
 using System.Collections;
 using System.Net;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
 
 public class SocketCLient : MonoBehaviour {
 
     // Use this for initialization
 
     public GameObject lefthand;
     public GameObject righthand;
     public GameObject body;
     private float xPos = 10.0f;
     private float yPos = 10.0f;
     private float bxPos = 0;
     private float byPos = 0;
     private String newString;
 
     Thread receiveThread;
     UdpClient client;
     public int port;
 
     //info
 
     public string lastReceivedUDPPacket = "";
     public string allReceivedUDPPackets = "";
 
     void Start () {
         init();
     }
 
     void OnGUI(){
         Rect  rectObj=new Rect (40,10,200,400);
         
         GUIStyle  style  = new GUIStyle ();
         
         style .alignment  = TextAnchor.UpperLeft;
         
         GUI .Box (rectObj,"# UDPReceive\n127.0.0.1 "+port +" #\n"
                   
                   //+ "shell> nc -u 127.0.0.1 : "+port +" \n"
                   
                   + "\nLast Packet: \n"+ lastReceivedUDPPacket
                   
                   //+ "\n\nAll Messages: \n"+allReceivedUDPPackets
                   
                   ,style );
 
     }
 
     private void init(){
         print ("UPDSend.init()");
 
         port = 5065;
 
         print ("Sending to 127.0.0.1 : " + port);
 
         receiveThread = new Thread (new ThreadStart(ReceiveData));
         receiveThread.IsBackground = true;
         receiveThread.Start ();
 
     }
 
     private void ReceiveData(){
         client = new UdpClient (port);
         while (true) {
             try{
                 IPEndPoint anyIP = new IPEndPoint(IPAddress.Parse("127.0.0.1"), port);
                 byte[] data = client.Receive(ref anyIP);
 
                 string text = Encoding.UTF8.GetString(data);
                 print (">> " + text);
                 lastReceivedUDPPacket=text;
                 allReceivedUDPPackets=allReceivedUDPPackets+text;
                 if (text.Contains("leftx")){
                     newString = text.Substring(5, 4);
                     xPos = int.Parse(newString);
                 }
                 if (text.Contains("lefty")){
                     newString = text.Substring(5, 4);
                     yPos = int.Parse(newString);
                     // if (yPos > 140)
                     //     yPos = -yPos;
                 }
                 if (text.Contains("bodyx")){
                     newString = text.Substring(5, 4);
                     bxPos = int.Parse(newString);
                 }
                 if (text.Contains("bodyy")){
                     newString = text.Substring(5, 4);
                     byPos = int.Parse(newString);
                 }
                 //xPos = int.Parse(text);
                 //xPos *= 0.021818f;
             }catch(Exception e){
                 print (e.ToString());
             }
         }
     }
 
     public string getLatestUDPPacket(){
         allReceivedUDPPackets = "";
         return lastReceivedUDPPacket;
     }
     
     // Update is called once per frame
     void Update () {
 
         lefthand.transform.position = new Vector3(0, xPos , yPos);
         body.transform.position = new Vector3(bxPos, byPos, 0);
         // if(yPos < byPos)
         //     lefthand.transform.position = new Vector3(xPos, yPos , 0);
         // if(yPos > byPos)
         //     lefthand.transform.position = new Vector3(xPos, yPos , 0);    
         //lefthand.transform.Rotate(0, yPos,  -10, Space.World);
     }
 
     void OnApplicationQuit(){
         if (receiveThread != null) {
             receiveThread.Abort();
             Debug.Log(receiveThread.IsAlive); //must be false
         }
     }
 }

Edit:

I also tried to only rotate the y- axis of the arm. Maybe the script is wrong?

 targetPoint = new Vector3(target.transform.position.x, transform.position.y, target.transform.position.z) - transform.position;
 targetRotation = Quaternion.LookRotation (-targetPoint, Vector3.up);
 transform.rotation = Quaternion.Slerp(transform.rotation, targetRotation, Time.deltaTime * 2.0f);