Preview Improvements (#32)

* significant improvements to preview window, performance and features

CMakeLists.txt

@@ -38,7 +38,7 @@ else(UNIX)
 endif()
 add_definitions(-DYERFACE_DATA_DIR="${YERFACE_DATA_DIR}")
 
-set( YERFACE_MODULES src/EventLogger.cpp src/FaceDetector.cpp src/FaceMapper.cpp src/FaceTracker.cpp src/FFmpegDriver.cpp src/FrameServer.cpp src/ImageSequence.cpp src/Logger.cpp src/MarkerTracker.cpp src/MarkerType.cpp src/Metrics.cpp src/OutputDriver.cpp src/PreviewHUD.cpp src/SDLDriver.cpp src/SphinxDriver.cpp src/Status.cpp src/Utilities.cpp src/WorkerPool.cpp src/yer-face.cpp )
+set( YERFACE_MODULES src/EventLogger.cpp src/FaceDetector.cpp src/FaceMapper.cpp src/FaceTracker.cpp src/FFmpegDriver.cpp src/FrameServer.cpp src/Logger.cpp src/MarkerTracker.cpp src/MarkerType.cpp src/Metrics.cpp src/OutputDriver.cpp src/PreviewHUD.cpp src/SDLDriver.cpp src/SphinxDriver.cpp src/Status.cpp src/Utilities.cpp src/WorkerPool.cpp src/yer-face.cpp )
 
 include(CTest)
 

VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

ci/version.sh

@@ -22,10 +22,9 @@ PACKAGE_VERSION=$(cat "${BASEPATH}/VERSION")
 VERSION_EXACT=false
 GIT_TAG=$(git describe --tags --exact-match 2> /dev/null)
 if [ -n "${GIT_TAG}" ]; then
-	if [ "${PACKAGE_VERSION}" != "${GIT_TAG}" ]; then
+	if [ "v${PACKAGE_VERSION}" != "${GIT_TAG}" ]; then
 		_die "VERSION file (${PACKAGE_VERSION}) and Git tag (${GIT_TAG}) do not match."
 	fi
-	PACKAGE_VERSION="${GIT_TAG}"
 	VERSION_EXACT=true
 fi
 VERSION_STRING="${VERSION_STRING}-${PACKAGE_VERSION}"

data/yer-face-config.json

@@ -136,10 +136,6 @@
     },
     "SDLDriver": {
     },
-    "ImageSequence": {
-      "numWorkersPerCPU": 0.25,
-      "numWorkers": 0
-    },
     "PreviewHUD": {
       "numWorkersPerCPU": 0.5,
       "numWorkers": 0,

doc/CommandLineUsage.md

@@ -342,10 +342,14 @@ Important notes:
 		If true, will preview processed audio out the computer's sound device.
 ```
 
-### Generating a Preview Image Sequence
-_Use this flag to save the preview display to an image sequence._
+### Controlling Preview Mirror Mode (Selfie Reflection)
+_Use this flag to control horizontal reflection of the preview (aka "Mirror Mode")._
+
+Important notes:
+- Default behavior is to link Mirror Mode to Low Latency mode, because Low Latency is generally used in conjunction with a webcam.
+- Mirror Mode has no effect on the performance capture pipeline, nor on the output data. The reflection along the X axis affects the preview only.
 
 ```
-	--previewImgSeq
-		If set, is presumed to be the file name prefix of the output preview image sequence.
+	--previewMirror
+		If true, mirror mode (horizontal reflection) of the preview will be forced on. If false, mirror mode will be forced off. If "auto" or not specified, mirror mode will be enabled for lowLatency mode and disabled otherwise.
 ```

src/FaceDetector.cpp

@@ -141,14 +141,18 @@ FacialDetectionBox FaceDetector::getFacialDetection(FrameNumber frameNumber) {
 	return detection;
 }
 
-void FaceDetector::renderPreviewHUD(Mat previewFrame, FrameNumber frameNumber, int density) {
+void FaceDetector::renderPreviewHUD(Mat previewFrame, FrameNumber frameNumber, int density, bool mirrorMode) {
 	YerFace_MutexLock(detectionsMutex);
 	FacialDetectionBox detection = detections[frameNumber];
 	YerFace_MutexUnlock(detectionsMutex);
 
 	if(density > 1) {
 		if(detection.set) {
-			cv::rectangle(previewFrame, detection.boxNormalSize, Scalar(255, 255, 0), 1);
+			cv::Rect2d box = detection.boxNormalSize;
+			if(mirrorMode) {
+				box.x = previewFrame.size().width - box.x - box.width;
+			}
+			cv::rectangle(previewFrame, box, Scalar(255, 255, 0), 1, LINE_AA); // FIXME - proportional drawing
 		}
 	}
 }

src/FaceDetector.hpp

@@ -42,7 +42,7 @@ class FaceDetector {
 	FaceDetector(json config, Status *myStatus, FrameServer *myFrameServer);
 	~FaceDetector() noexcept(false);
 	FacialDetectionBox getFacialDetection(FrameNumber frameNumber);
-	void renderPreviewHUD(cv::Mat previewFrame, FrameNumber frameNumber, int density);
+	void renderPreviewHUD(cv::Mat previewFrame, FrameNumber frameNumber, int density, bool mirrorMode);
 private:
 	void doDetectFace(WorkerPoolWorker *worker, FaceDetectionTask task);
 	static void handleFrameStatusChange(void *userdata, WorkingFrameStatus newStatus, FrameTimestamps frameTimestamps);

src/FaceMapper.cpp

@@ -131,32 +131,37 @@ FaceMapper::~FaceMapper() noexcept(false) {
 	delete logger;
 }
 
-void FaceMapper::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int density) {
+void FaceMapper::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int density, bool mirrorMode) {
 	for(MarkerTracker *markerTracker : trackers) {
-		markerTracker->renderPreviewHUD(frame, frameNumber, density);
+		markerTracker->renderPreviewHUD(frame, frameNumber, density, mirrorMode);
 	}
 	if(density > 0) {
 		int gridIncrement = 15; //FIXME - magic numbers
 		Rect2d previewRect;
 		Point2d previewCenter;
 		previewHUD->createPreviewHUDRectangle(frame.size(), &previewRect, &previewCenter);
-		double previewPointScale = previewRect.width / 200;
 		rectangle(frame, previewRect, Scalar(20, 20, 20), FILLED);
+		//// FIXME - grid should be related to real-world units if possible
 		if(density > 4) {
 			for(int x = (int)previewRect.x; x < (int)(previewRect.x + previewRect.width); x = x + gridIncrement) {
-				cv::line(frame, Point2d(x, previewRect.y), Point2d(x, previewRect.y + previewRect.height), Scalar(75, 75, 75));
+				cv::line(frame, Point2d(x, previewRect.y), Point2d(x, previewRect.y + previewRect.height), Scalar(75, 75, 75), 1, LINE_AA); // FIXME - proportional drawing
 			}
 			for(int y = (int)previewRect.y; y < (int)(previewRect.y + previewRect.height); y = y + gridIncrement) {
-				cv::line(frame, Point2d(previewRect.x, y), Point2d(previewRect.x + previewRect.width, y), Scalar(75, 75, 75));
+				cv::line(frame, Point2d(previewRect.x, y), Point2d(previewRect.x + previewRect.width, y), Scalar(75, 75, 75), 1, LINE_AA); // FIXME - proportional drawing
 			}
 		}
+		double previewPointScale = previewRect.width / 200; // FIXME - more magic numbers?
+		double mirrorFlip = 1.0;
+		if(mirrorMode) {
+			mirrorFlip = -1.0;
+		}
 		for(MarkerTracker *markerTracker : trackers) {
 			MarkerPoint markerPoint = markerTracker->getMarkerPoint(frameNumber);
 			if(markerPoint.set) {
 				Point2d previewPoint = Point2d(
-						(markerPoint.point3d.x * previewPointScale) + previewCenter.x,
+						(markerPoint.point3d.x * previewPointScale * mirrorFlip) + previewCenter.x,
 						(markerPoint.point3d.y * previewPointScale) + previewCenter.y);
-				Utilities::drawX(frame, previewPoint, Scalar(255, 255, 255));
+				Utilities::drawX(frame, previewPoint, Scalar(255, 255, 255)); // FIXME - proportional drawing
 			}
 		}
 	}

src/FaceMapper.hpp

@@ -26,7 +26,7 @@ class FaceMapper {
 public:
 	FaceMapper(json config, Status *myStatus, FrameServer *myFrameServer, FaceTracker *myFaceTracker, PreviewHUD *myPreviewHUD);
 	~FaceMapper() noexcept(false);
-	void renderPreviewHUD(cv::Mat frame, FrameNumber frameNumber, int density);
+	void renderPreviewHUD(cv::Mat frame, FrameNumber frameNumber, int density, bool mirrorMode);
 	FrameServer *getFrameServer(void);
 	FaceTracker *getFaceTracker(void);
 private:

src/FaceTracker.cpp

@@ -456,7 +456,7 @@ bool FaceTracker::doConvertLandmarkPointToImagePoint(DlibPointPointer pointPoint
 	return true;
 }
 
-void FaceTracker::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int density) {
+void FaceTracker::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int density, bool mirrorMode) {
 	YerFace_MutexLock(myMutex);
 	if(frameNumber < 0 || outputFrames.find(frameNumber) == outputFrames.end()) {
 		YerFace_MutexUnlock(myMutex);
@@ -469,6 +469,8 @@ void FaceTracker::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int densi
 	FacialCameraModel camera = facialCameraModel;
 	YerFace_MutexUnlock(myAssignmentMutex);
 
+	int frameWidth = frame.size().width;
+
 	if(density > 0) {
 		if(output.facialPose.set) {
 			std::vector<Point3d> gizmo3d(6);
@@ -483,15 +485,23 @@ void FaceTracker::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int densi
 			Mat tempRotationVector;
 			Rodrigues(output.facialPose.rotationMatrix, tempRotationVector);
 			projectPoints(gizmo3d, tempRotationVector, output.facialPose.translationVector, camera.cameraMatrix, camera.distortionCoefficients, gizmo2d);
-			arrowedLine(frame, gizmo2d[0], gizmo2d[1], Scalar(0, 0, 255), 2);
-			arrowedLine(frame, gizmo2d[2], gizmo2d[3], Scalar(255, 0, 0), 2);
-			arrowedLine(frame, gizmo2d[4], gizmo2d[5], Scalar(0, 255, 0), 2);
+			if(mirrorMode) {
+				for(int i = 0; i <= 5; i++) {
+					gizmo2d[i].x = frameWidth - gizmo2d[i].x;
+				}
+			}
+			arrowedLine(frame, gizmo2d[0], gizmo2d[1], Scalar(0, 0, 255), 2, LINE_AA); // FIXME - proportional drawing size
+			arrowedLine(frame, gizmo2d[2], gizmo2d[3], Scalar(255, 0, 0), 2, LINE_AA); // FIXME - proportional drawing size
+			arrowedLine(frame, gizmo2d[4], gizmo2d[5], Scalar(0, 255, 0), 2, LINE_AA); // FIXME - proportional drawing size
 		}
 	}
 	if(density > 3) {
 		if(output.facialFeatures.set) {
-			for(auto feature : output.facialFeatures.featuresExposed.features) {
-				Utilities::drawX(frame, feature, Scalar(147, 20, 255));
+			for(Point2d feature : output.facialFeatures.featuresExposed.features) {
+				if(mirrorMode) {
+					feature.x = frameWidth - feature.x;
+				}
+				Utilities::drawX(frame, feature, Scalar(147, 20, 255)); // FIXME - proportional drawing
 			}
 		}
 	}
@@ -519,7 +529,11 @@ void FaceTracker::renderPreviewHUD(Mat frame, FrameNumber frameNumber, int densi
 			projectPoints(edges3d, output.facialPose.rotationMatrix, output.facialPose.translationVector, camera.cameraMatrix, camera.distortionCoefficients, edges2d);
 
 			for(unsigned int i = 0; i + 1 < edges2d.size(); i = i + 2) {
-				cv::line(frame, edges2d[i], edges2d[i + 1], Scalar(255, 255, 255));
+				if(mirrorMode) {
+					edges2d[i].x = frameWidth - edges2d[i].x;
+					edges2d[i + 1].x = frameWidth - edges2d[i + 1].x;
+				}
+				cv::line(frame, edges2d[i], edges2d[i + 1], Scalar(255, 255, 255), 1, LINE_AA); // FIXME - proportional drawing
 			}
 		}
 	}

src/FaceTracker.hpp

@@ -137,7 +137,7 @@ class FaceTracker {
 public:
 	FaceTracker(json config, Status *myStatus, SDLDriver *mySDLDriver, FrameServer *myFrameServer, FaceDetector *myFaceDetector);
 	~FaceTracker() noexcept(false);
-	void renderPreviewHUD(cv::Mat frame, FrameNumber frameNumber, int density);
+	void renderPreviewHUD(cv::Mat frame, FrameNumber frameNumber, int density, bool mirrorMode);
 	FacialFeatures getFacialFeatures(FrameNumber frameNumber);
 	FacialCameraModel getFacialCameraModel(void);
 	FacialPose getFacialPose(FrameNumber frameNumber);

src/FrameServer.cpp

@@ -40,6 +40,7 @@ FrameServer::FrameServer(json config, Status *myStatus, bool myLowLatency) {
 	metrics = new Metrics(config, "FrameServer");
 
 	draining = false;
+	mirrorMode = false;
 	workerPool = NULL;
 
 	WorkerPoolParameters workerPoolParameters;
@@ -127,7 +128,11 @@ void FrameServer::insertNewFrame(VideoFrame *videoFrame) {
 	}
 
 	workingFrame->frame = videoFrame->frameCV.clone();
-	workingFrame->previewFrame = workingFrame->frame.clone();
+	if(mirrorMode) {
+		cv::flip(workingFrame->frame, workingFrame->previewFrame, 1);
+	} else {
+		workingFrame->previewFrame = workingFrame->frame.clone();
+	}
 
 	frameSize = workingFrame->frame.size();
 	frameSizeSet = true;
@@ -189,6 +194,12 @@ void FrameServer::setDraining(void) {
 	YerFace_MutexUnlock(myMutex);
 }
 
+void FrameServer::setMirrorMode(bool myMirrorMode) {
+	YerFace_MutexLock(myMutex);
+	mirrorMode = myMirrorMode;
+	YerFace_MutexUnlock(myMutex);
+}
+
 WorkingFrame *FrameServer::getWorkingFrame(FrameNumber frameNumber) {
 	YerFace_MutexLock(myMutex);
 	auto frameIter = frameStore.find(frameNumber);

src/FrameServer.hpp

@@ -30,11 +30,10 @@ enum WorkingFrameStatus: unsigned int {
 	FRAME_STATUS_DETECTION = 2, //Primary face rectangle is being identified by FaceDetector
 	FRAME_STATUS_TRACKING = 3, //Face raw landmarks and pose are being recovered by FaceTracker
 	FRAME_STATUS_MAPPING = 4, //Primary markers position is being recovered by FaceMapper and its children.
-	FRAME_STATUS_PREVIEW_RENDER = 5, //Frame preview is to be rendered.
-	FRAME_STATUS_PREVIEW_DISPLAY = 6, //Frame preview is to be displayed. (NOTE: After this stage, ALL bitmap data associated with this frame is RELEASED!)
-	FRAME_STATUS_LATE_PROCESSING = 7, //Frame is eligible for any late-stage processing (like Sphinx data or event logs).
-	FRAME_STATUS_DRAINING = 8, //Last call before this frame is gone. (Output frame data.)
-	FRAME_STATUS_GONE = 9 //This frame is about to be freed and purged from the frame store. (No checkpoints can be registered for this status!)
+	FRAME_STATUS_PREVIEW_DISPLAY = 5, //Frame preview is to be displayed. (NOTE: After this stage, ALL bitmap data associated with this frame is RELEASED!)
+	FRAME_STATUS_LATE_PROCESSING = 6, //Frame is eligible for any late-stage processing (like Sphinx data or event logs).
+	FRAME_STATUS_DRAINING = 7, //Last call before this frame is gone. (Output frame data.)
+	FRAME_STATUS_GONE = 8 //This frame is about to be freed and purged from the frame store. (No checkpoints can be registered for this status!)
 };
 
 class WorkingFrame {
@@ -68,6 +67,7 @@ class FrameServer {
 	FrameServer(json config, Status *myStatus, bool myLowLatency);
 	~FrameServer() noexcept(false);
 	void setDraining(void);
+	void setMirrorMode(bool myMirrorMode);
 	void onFrameServerDrainedEvent(FrameServerDrainedEventCallback callback);
 	void onFrameStatusChangeEvent(FrameStatusChangeEventCallback callback);
 	void registerFrameStatusCheckpoint(WorkingFrameStatus status, string checkpointKey);
@@ -85,6 +85,7 @@ class FrameServer {
 	Status *status;
 	bool lowLatency;
 	bool draining;
+	bool mirrorMode;
 	int detectionBoundingBox;
 	double detectionScaleFactor;
 	Logger *logger;