- Home /
Recording hand gestures and playing it back on same hand model
I am using a Leap Motion sensor in Unity to record all the hand and finger joint rotation data into a .csv file. This file is then read and used to animate the same 3D hand, but the resulting hand animation is very crooked, it becomes all over the place and deformed
using System;
using System.Collections.Generic;
using UnityEngine;
/*
*The author of this script is Ashwin Aby Philip
* This script is used to record all joint angle rotation data available in the hand model provided by Leap Motion
*/
public class RecordData : MonoBehaviour
{
public GameObject rWrist;
public GameObject rPalm;
public GameObject indexMeta;
public GameObject thumbMeta;
public GameObject middleMeta;
public GameObject ringMeta;
public GameObject pinkyMeta;
public GameObject index_1;
public GameObject index_2;
public GameObject index_3;
public GameObject indexEnd;
public GameObject thumb_1;
public GameObject thumb_2;
public GameObject thumbEnd;
public GameObject middle_1;
public GameObject middle_2;
public GameObject middle_3;
public GameObject middleEnd;
public GameObject ring_1;
public GameObject ring_2;
public GameObject ring_3;
public GameObject ringEnd;
public GameObject pinky_1;
public GameObject pinky_2;
public GameObject pinky_3;
public GameObject pinkyEnd;
List<long> timestampList = new List<long>();
List<Quaternion> index1List = new List<Quaternion>();
List<Quaternion> thumb1List = new List<Quaternion>();
List<Quaternion> middle1List = new List<Quaternion>();
List<Quaternion> ring1List = new List<Quaternion>();
List<Quaternion> pinky1List = new List<Quaternion>();
List<Quaternion> index2List = new List<Quaternion>();
List<Quaternion> thumb2List = new List<Quaternion>();
List<Quaternion> middle2List = new List<Quaternion>();
List<Quaternion> ring2List = new List<Quaternion>();
List<Quaternion> pinky2List = new List<Quaternion>();
List<Quaternion> index3List = new List<Quaternion>();
List<Quaternion> middle3List = new List<Quaternion>();
List<Quaternion> ring3List = new List<Quaternion>();
List<Quaternion> pinky3List = new List<Quaternion>();
List<Quaternion> indexEndList = new List<Quaternion>();
List<Quaternion> thumbEndList = new List<Quaternion>();
List<Quaternion> middleEndList = new List<Quaternion>();
List<Quaternion> ringEndList = new List<Quaternion>();
List<Quaternion> pinkyEndList = new List<Quaternion>();
List<Quaternion> indexMetaList = new List<Quaternion>();
List<Quaternion> thumbMetaList = new List<Quaternion>();
List<Quaternion> middleMetaList = new List<Quaternion>();
List<Quaternion> ringMetaList = new List<Quaternion>();
List<Quaternion> pinkyMetaList = new List<Quaternion>();
List<Quaternion> rWristList = new List<Quaternion>();
List<Quaternion> rPalmList = new List<Quaternion>();
Dictionary<string, List<Quaternion>> store = new Dictionary<string, List<UnityEngine.Quaternion>>();
// Start is called before the first frame update
void Start()
{
}
// Update is called once per frame
void Update()
{
//get current system time in UNIX and record in list
long unixTimestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
timestampList.Add(unixTimestamp);
//record all quaternion rotation into lists
Debug.Log("local rotation pinky3 z " + pinky_3.transform.localEulerAngles.z + " rotation pinky3 z " + pinky_3.transform.rotation.z * Mathf.Rad2Deg);
thumb1List.Add(thumb_1.transform.localRotation);
index1List.Add(index_1.transform.localRotation);
middle1List.Add(middle_1.transform.localRotation);
ring1List.Add(ring_1.transform.localRotation);
pinky1List.Add(pinky_1.transform.localRotation);
thumb2List.Add(thumb_2.transform.localRotation);
index2List.Add(index_2.transform.localRotation);
middle2List.Add(middle_2.transform.localRotation);
ring2List.Add(ring_2.transform.localRotation);
pinky2List.Add(pinky_2.transform.localRotation);
index3List.Add(index_3.transform.localRotation);
middle3List.Add(middle_3.transform.localRotation);
ring3List.Add(ring_3.transform.localRotation);
pinky3List.Add(pinky_3.transform.localRotation);
indexEndList.Add(indexEnd.transform.localRotation);
middleEndList.Add(middleEnd.transform.localRotation);
thumbEndList.Add(thumbEnd.transform.localRotation);
ringEndList.Add(ringEnd.transform.localRotation);
pinkyEndList.Add(pinkyEnd.transform.localRotation);
indexMetaList.Add(indexMeta.transform.localRotation);
thumbMetaList.Add(thumbMeta.transform.localRotation);
middleMetaList.Add(middleMeta.transform.localRotation);
ringMetaList.Add(ringMeta.transform.localRotation);
pinkyMetaList.Add(pinkyMeta.transform.localRotation);
rWristList.Add(rWrist.transform.localRotation);
rPalmList.Add(rPalm.transform.localRotation);
}
private void OnApplicationQuit()
{
using (System.IO.StreamWriter file =
new System.IO.StreamWriter(@"C:\Users\a_phi\Documents\New Unity Project (4)\Assets\test final.csv"))
{
//When application has finished, the lists containing the data is added to the Dictionary called store
store.Add("rWrist", rWristList);
store.Add("rPalm", rPalmList);
store.Add("thumbMeta", thumbMetaList);
store.Add("thumb_1", thumb1List);
store.Add("thumb_2", thumb2List);
store.Add("thumbEnd", thumbEndList);
store.Add("indexMeta", indexMetaList);
store.Add("index_1",index1List);
store.Add("index_2", index2List);
store.Add("index_3", index3List);
store.Add("indexEnd", indexEndList);
store.Add("middleMeta", middleMetaList);
store.Add("middle_1", middle1List);
store.Add("middle_2", middle2List);
store.Add("middle_3", middle3List);
store.Add("middleEnd", middleEndList);
store.Add("ringMeta", ringMetaList);
store.Add("ring_1", ring1List);
store.Add("ring_2", ring2List);
store.Add("ring_3", ring3List);
store.Add("ringEnd", ringEndList);
store.Add("pinkyMeta", pinkyMetaList);
store.Add("pinky_1", pinky1List);
store.Add("pinky_2", pinky2List);
store.Add("pinky_3", pinky3List);
store.Add("pinkyEnd",pinkyEndList);
//name of joints to find corresponding lists when looping through
string[] joints = new[]{ "rWrist" , "rPalm", "thumbMeta", "thumb_1", "thumb_2", "thumbEnd",
"indexMeta", "index_1", "index_2", "index_3", "indexEnd", "middleMeta", "middle_1",
"middle_2", "middle_3", "middleEnd", "ringMeta", "ring_1", "ring_2", "ring_3", "ringEnd",
"pinkyMeta", "pinky_1", "pinky_2", "pinky_3", "pinkyEnd"};
//loop through Dictionary and write each Quaternion angle of each joint to file
for (int i = 0; i < store["index_1"].Capacity; i++)
{
if(i == 0)
{
//write column headings for .csv file
string colHeadings = "Timestamp, Wrist x, Wrist y, Wrist z," +
"Palm x, Palm y, Palm z, Thumb Meta x, Thumb Meta y, Thumb Meta z, Thumb01 x, Thumb01 y, Thumb01 z, " +
"Thumb02 x, Thumb02 y, Thumb02 z, Thumb End x, Thumb End y, Thumb End z," +
"Index Meta x,Index Meta y,Index Meta z, Index01 x, Index01 y, Index01 z, Index02 x,Index02 y,Index02 z, " +
"Index03 x,Index03 y,Index03 z, Index End x, Index End y, Index End z," +
"Middle Meta x, Middle Meta y, Middle Meta z, Middle01 x, Middle01 y, Middle01 z, Middle02 x, Middle02 y, Middle02 z, " +
"Middle03 x, Middle03 y, Middle03 z, Middle End x, Middle End y, Middle End z," +
"Ring Meta x, Ring Meta y, Ring Meta z, Ring01 x, Ring01 y, Ring01 z, Ring02 x, Ring02 y, Ring02 z, " +
"Ring03 x, Ring03 y, Ring03 z, Ring End x, Ring End y, Ring End z," +
"Pinky Meta x, Pinky Meta y, Pinky Meta z, Pinky01 x, Pinky01 y, Pinky01 z, Pinky02 x, Pinky02 y, Pinky02 z, " +
"Pinky03 x, Pinky03 y, Pinky03 z, Pinky End x, Pinky End y, Pinky End z";
file.WriteLine(colHeadings);
}
string toAppend = null;
foreach (String theJoint in joints)
{
//quaternion of joint is obtained by iterative index
List<Quaternion> entries = store[theJoint];
Quaternion entryValue = entries[i];
//individual angle is obtained and converted from radians to degrees
float entryValueX = Math.Abs(entryValue.x*Mathf.Rad2Deg);
float entryValueY = Math.Abs(entryValue.y*Mathf.Rad2Deg);
float entryValueZ = Math.Abs(entryValue.z*Mathf.Rad2Deg);
toAppend += entryValueX + ", " + entryValueY + ", " + entryValueZ + ", ";
}
toAppend.Remove(toAppend.Length-2);
//write line to file with timestamp
file.WriteLine(timestampList[i] + ", " + toAppend);
}
}
}
}
/*
*The Author of this script is Ashwin Aby Philip. This script is used to read csv files with the
* appropriate leap motion joint angle data and transform this to a 3d hand model
*/
using System.Collections;
using UnityEngine;
namespace CsvNameSpace {
public class GetterSetter {
public bool animate;
public bool Animate
{
get { return animate; }
set { animate = true; }
}
}
public class CsvReader : MonoBehaviour
{
public string[] fieldsNext;
public string[] fields;
// public GameObject armature;
public TextAsset csvFile;
// public GameObject forearm;
public GameObject rWrist;
public GameObject rPalm;
public GameObject indexMeta;
public GameObject thumbMeta;
public GameObject middleMeta;
public GameObject ringMeta;
public GameObject pinkyMeta;
public GameObject index_1;
public GameObject index_2;
public GameObject index_3;
public GameObject indexEnd;
public GameObject thumb_1;
public GameObject thumb_2;
public GameObject thumbEnd;
public GameObject middle_1;
public GameObject middle_2;
public GameObject middle_3;
public GameObject middleEnd;
public GameObject ring_1;
public GameObject ring_2;
public GameObject ring_3;
public GameObject ringEnd;
public GameObject pinky_1;
public GameObject pinky_2;
public GameObject pinky_3;
public GameObject pinkyEnd;
// Update is called once per frame
int i = 0;
long previoustime = 0;
void Update()
{
StartCoroutine(SetAngles());
i++;
}
IEnumerator SetAngles()
{
print("Time " + Time.time);
if (i > 0)
{
string[] records = csvFile.text.Split('\n');
if (i < records.Length - 2)
{
fields = records[i].Split(',');
fieldsNext = records[i + 1].Split(',');
}
else
{
//restart when reached the end of file and no more rows are remaining
// print("i = " + i + " length = " + records.Length);
//ignore first row as that contains headings and not data
i = 1;
//looks at current row of data
fields = records[i].Split(',');
//looks at the next row of data
fieldsNext = records[i + 1].Split(',');
}
//each joint angle is transformed from the data in the CSV file
rWrist.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[1]), float.Parse(fields[2]), float.Parse(fields[3])));
rPalm.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[4]), float.Parse(fields[5]), float.Parse(fields[6])));
thumbMeta.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[7]), float.Parse(fields[8]), float.Parse(fields[9])));
thumb_1.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[10]), float.Parse(fields[11]), float.Parse(fields[12])));
thumb_2.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[13]), float.Parse(fields[14]), float.Parse(fields[15])));
// thumb_3.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[16]), float.Parse(fields[17]), float.Parse(fields[18])));
thumbEnd.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[16]), float.Parse(fields[17]), float.Parse(fields[18])));
indexMeta.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[19]), float.Parse(fields[20]), float.Parse(fields[21])));
index_1.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[22]), float.Parse(fields[23]), float.Parse(fields[24])));
index_2.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[25]), float.Parse(fields[26]), float.Parse(fields[27])));
index_3.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[28]), float.Parse(fields[29]), float.Parse(fields[30])));
indexEnd.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[31]), float.Parse(fields[32]), float.Parse(fields[33])));
middleMeta.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[34]), float.Parse(fields[35]), float.Parse(fields[36])));
middle_1.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[37]), float.Parse(fields[38]), float.Parse(fields[39])));
middle_2.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[40]), float.Parse(fields[41]), float.Parse(fields[42])));
middle_3.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[43]), float.Parse(fields[44]), float.Parse(fields[45])));
middleEnd.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[46]), float.Parse(fields[47]), float.Parse(fields[48])));
ringMeta.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[49]), float.Parse(fields[50]), float.Parse(fields[51])));
ring_1.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[52]), float.Parse(fields[53]), float.Parse(fields[54])));
ring_2.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[55]), float.Parse(fields[56]), float.Parse(fields[57])));
ring_3.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[58]), float.Parse(fields[59]), float.Parse(fields[60])));
ringEnd.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[61]), float.Parse(fields[62]), float.Parse(fields[63])));
Debug.Log("ring_2 value z" + float.Parse(fields[57]));
pinkyMeta.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[64]), float.Parse(fields[65]), float.Parse(fields[66])));
pinky_1.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[67]), float.Parse(fields[68]), float.Parse(fields[69])));
pinky_2.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[70]), float.Parse(fields[71]), float.Parse(fields[72])));
pinky_3.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[73]), float.Parse(fields[74]), float.Parse(fields[75])));
pinkyEnd.transform.localRotation = Quaternion.Euler(new Vector3(float.Parse(fields[76]), float.Parse(fields[77]), float.Parse(fields[78])));
long currentTime = long.Parse(fields[0]);
long nextTime = long.Parse(fieldsNext[0]);
//calculate the delay required between the current timestamp of frame and the previous timestamp
yield return new WaitForSeconds(nextTime - currentTime);
//now simulates animation in real time
}
}
}
}
I record the transform.localRotation of each joint and store it as a Quaternion. The x,y,z of the quaternion is then converted to degrees and written to a file. However, the hand movements become very deformed when i read it into the same hand model when played back.
Is there something wrong with using Quaternions? Please find my scripts attached. I believe I have all the correct columns of my csv file correctly aligned in both scripts to be read.
Answer by madhawaperera · Sep 28, 2020 at 03:32 AM
Hi @unity_lF98_nVJe9di1A looks like we're trying to crack the same nut here... have you made any progress since this post? Appreciate it if you have an update on where you ended up eventually :D