- Home /
how to optimize real-time stitching by OpenCV
Hello everyone! I am trying to make a real-time stitching function from camera in Unity. After watching some tutorials, I have success to make a simple real-time stitching from camera and this is my work. However, you may find there are some problems in the video. First, there is shaking every frame from the stitched video; second is the stitched video color is different from the left-hand side; and finally, there is a conspicuous line from the stitched video. Therefore, I want to know how to fix the problem and thanks everyone who seeing my question. Here is my code of the project, and I am using OpenCVforUnity for this project.
void OnStitch1()
{
pr2 = Stitching(second_mat_gray, first_mat_gray);
Utils.matToTexture2D(pr2, _Texture1);
output.texture = _Texture1;
output.GetComponent<AspectRatioFitter>().aspectRatio = (float)_Texture1.width / (float)_Texture1.height;
}
Mat Stitching(Mat first_mat_temp, Mat second_mat_temp)
{
//finding the keypoint between two streaming
detector.detect(first_mat_temp, keypointsSrc);
extractor.compute(first_mat_temp, keypointsSrc, descriptorsSrc);
detector.detect(second_mat_temp, keypointsScene);
extractor.compute(second_mat_temp, keypointsScene, descriptorsScene);
//Matching the keypoints
matcher.clear();
matcher.match(descriptorsSrc, descriptorsScene, matches);
matchesList = matches.toList();
//After matching, trying to find the good match point by distance
for (int i = 0; i < descriptorsSrc.rows(); i++)
{
dist = (double)matchesList[i].distance;
if (dist < min_dist)
min_dist = dist;
if (dist > max_dist)
max_dist = dist;
}
good_matches.Clear();
for (int i = 0; i < matchesList.Count; i++)
{
if (matchesList[i].distance < 10 * min_dist)
good_matches.Add(matchesList[i]);
}
GM = good_matches.Count;
//Adding Keypoint to List for finding Homography with the OpenCV function
keypoints_objectList = keypointsSrc.toList();
keypoints_sceneList = keypointsScene.toList();
keypointsSrc.release();
keypointsScene.release();
for (int i = 0; i < good_matches.Count; i++)
{
objList.Add(keypoints_objectList[good_matches[i].queryIdx].pt);
sceneList.Add(keypoints_sceneList[good_matches[i].trainIdx].pt);
}
MatOfPoint2f obj = new MatOfPoint2f();
MatOfPoint2f scene = new MatOfPoint2f();
obj.fromList(objList);
scene.fromList(sceneList);
H = Calib3d.findHomography(obj, scene, Calib3d.RANSAC, 10);
//Start warp the streaming from second camera to connect the streaming from first camera
//Trying to stop the shaking, but not smooth enough
warp_mat = second_mat.clone();
if (GM - gmCount > 10)
{
gmCount = good_matches.Count;
tempH = H.clone();
Imgproc.warpPerspective(second_mat, warp_mat, H, ims);
}
else
{
Imgproc.warpPerspective(second_mat, warp_mat, tempH, ims);
}
if (gmCount > 400 || GM < 100)
gmCount = GM;
//Last Step for stitching as a whole Mat
half = new Mat(warp_mat, _rect);
first_mat.copyTo(half);
return warp_mat;
}
I find $$anonymous$$ Blending may be the solution to clear the conspicuous line from the stitched video, but I have no idea how to do that in Unity, does anyone can giving me some suggestion?
Hey did you find anything PS. Which version of Unity were you using, on 2020.2, I am getting constant crashes even on this code.