feat: Modified face_mesh_lib functionality

Change List:
- Combined `MPFaceMeshDetector::GetFaceCount` and `MPFaceMeshDetector::GetFaceLandmarks` into `MPFaceMeshDetector::ProcessFrame2D`
- Added `MPFaceMeshDetectorLandmarksNum` that exports number of face landmarks detected by MediaPipe

mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp

@@ -28,15 +28,15 @@ int main(int argc, char **argv) {
   constexpr char face_landmark_model_path[] =
       "mediapipe/modules/face_landmark/face_landmark.tflite";
 
-  MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(
+  MPFaceMeshDetector *faceMeshDetector = MPFaceMeshDetectorConstruct(
       maxNumFaces, face_detection_model_path, face_landmark_model_path);
 
-  // allocate memory for face landmarks
+  // Allocate memory for face landmarks.
   auto multiFaceLandmarks = new cv::Point2f *[maxNumFaces];
-  constexpr auto mediapipeFaceLandmarksNum = 468;
   for (int i = 0; i < maxNumFaces; ++i) {
-    multiFaceLandmarks[i] = new cv::Point2f[mediapipeFaceLandmarksNum];
+    multiFaceLandmarks[i] = new cv::Point2f[MPFaceMeshDetectorLandmarksNum];
   }
+  const auto faceCount = std::make_unique<int>();
 
   LOG(INFO) << "FaceMeshDetector constructed.";
 
@@ -56,21 +56,19 @@ int main(int argc, char **argv) {
     cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
     cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
 
-    int faceCount =
+    MPFaceMeshDetectorProcessFrame2D(faceMeshDetector, camera_frame,
-        FaceMeshDetector_GetFaceCount(faceMeshDetector, camera_frame);
+                                     faceCount.get(), multiFaceLandmarks);
 
-    LOG(INFO) << "Detected faces num: " << faceCount;
+    LOG(INFO) << "Detected faces num: " << *faceCount;
-
-    if (faceCount > 0) {
-
-      FaceMeshDetector_GetFaceLandmarks(faceMeshDetector, multiFaceLandmarks);
 
+    if (*faceCount > 0) {
       auto &face_landmarks = multiFaceLandmarks[0];
       auto &landmark = face_landmarks[0];
 
       LOG(INFO) << "First landmark: x - " << landmark.x << ", y - "
                 << landmark.y;
     }
+
     const int pressed_key = cv::waitKey(5);
     if (pressed_key >= 0 && pressed_key != 255)
       grab_frames = false;
@@ -80,11 +78,11 @@ int main(int argc, char **argv) {
 
   LOG(INFO) << "Shutting down.";
 
-  // deallocate memory for face landmarks
+  // Deallocate memory for face landmarks.
   for (int i = 0; i < maxNumFaces; ++i) {
     delete[] multiFaceLandmarks[i];
   }
   delete[] multiFaceLandmarks;
 
-  FaceMeshDetector_Destruct(faceMeshDetector);
+  MPFaceMeshDetectorDestruct(faceMeshDetector);
 }

mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp

@@ -1,9 +1,5 @@
-#include <windows.h>
-
 #include "face_mesh_lib.h"
 
-#define DEBUG
-
 MPFaceMeshDetector::MPFaceMeshDetector(int numFaces,
                                        const char *face_detection_model_path,
                                        const char *face_landmark_model_path) {
@@ -19,9 +15,7 @@ absl::Status
 MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
                                          const char *face_detection_model_path,
                                          const char *face_landmark_model_path) {
-  if (numFaces <= 0) {
+  numFaces = std::max(numFaces, 1);
-    numFaces = 1;
-  }
 
   if (face_detection_model_path == nullptr) {
     face_detection_model_path =
@@ -33,6 +27,7 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
         "mediapipe/modules/face_landmark/face_landmark.tflite";
   }
 
+  // Prepare graph config.
   auto preparedGraphConfig = absl::StrReplaceAll(
       graphConfig, {{"$numFaces", std::to_string(numFaces)}});
   preparedGraphConfig = absl::StrReplaceAll(
@@ -70,8 +65,11 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
   return absl::OkStatus();
 }
 
-absl::Status
+absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus(
-MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
+    const cv::Mat &camera_frame, int *numFaces,
+    cv::Point2f **multi_face_landmarks) {
+  *numFaces = 0;
+
   // Wrap Mat into an ImageFrame.
   auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
       mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
@@ -85,6 +83,8 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
   MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
       kInputStream, mediapipe::Adopt(input_frame.release())
                         .At(mediapipe::Timestamp(frame_timestamp_us))));
+
+  // Get face count.
   mediapipe::Packet face_count_packet;
   if (!face_count_poller_ptr ||
       !face_count_poller_ptr->Next(&face_count_packet)) {
@@ -94,29 +94,11 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
 
   auto &face_count = face_count_packet.Get<int>();
 
-  faceCount = face_count;
+  if (face_count <= 0) {
-
     return absl::OkStatus();
-}
-
-int MPFaceMeshDetector::GetFaceCount(const cv::Mat &camera_frame) {
-  const auto status = GetFaceCountWithStatus(camera_frame);
-  if (!status.ok()) {
-    LOG(INFO) << "Failed GetFaceCount.";
-    LOG(INFO) << status.message();
-  }
-
-  return faceCount;
-}
-
-absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
-    cv::Point2f **multi_face_landmarks) {
-
-  if (faceCount <= 0) {
-    return absl::CancelledError(
-        "Failed during gettinglandmarks, because faceCount is <= 0.");
   }
 
+  // Get face landmarks.
   mediapipe::Packet face_landmarks_packet;
   if (!landmarks_poller_ptr ||
       !landmarks_poller_ptr->Next(&face_landmarks_packet)) {
@@ -127,9 +109,15 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
       face_landmarks_packet
           .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>();
 
-  for (int i = 0; i < faceCount; ++i) {
+  // Convert landmarks to cv::Point2f**.
+  for (int i = 0; i < face_count; ++i) {
     const auto &normalizedLandmarkList = output_landmarks_vector[i];
     const auto landmarks_num = normalizedLandmarkList.landmark_size();
+
+    if (landmarks_num != kLandmarksNum) {
+      return absl::CancelledError("Detected unexpected landmarks number.");
+    }
+
     auto &face_landmarks = multi_face_landmarks[i];
 
     for (int j = 0; j < landmarks_num; ++j) {
@@ -139,47 +127,43 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
     }
   }
 
-  faceCount = -1;
+  *numFaces = face_count;
 
   return absl::OkStatus();
 }
 
-void MPFaceMeshDetector::GetFaceLandmarks(cv::Point2f **multi_face_landmarks) {
+void MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame,
-  const auto status = GetFaceLandmarksWithStatus(multi_face_landmarks);
+                                        int *numFaces,
+                                        cv::Point2f **multi_face_landmarks) {
+  const auto status =
+      ProcessFrame2DWithStatus(camera_frame, numFaces, multi_face_landmarks);
   if (!status.ok()) {
-    LOG(INFO) << "Failed GetFaceLandmarks.";
+    LOG(INFO) << "Failed ProcessFrame2D.";
     LOG(INFO) << status.message();
   }
 }
 
 extern "C" {
 DLLEXPORT MPFaceMeshDetector *
-FaceMeshDetector_Construct(int numFaces, const char *face_detection_model_path,
+MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path,
                             const char *face_landmark_model_path) {
   return new MPFaceMeshDetector(numFaces, face_detection_model_path,
                                 face_landmark_model_path);
 }
 
-DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) {
+DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector) {
   delete detector;
 }
 
-DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector,
-                                            const cv::Mat &camera_frame) {
-  return detector->GetFaceCount(camera_frame);
-}
-
 DLLEXPORT void
-FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector,
+MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector,
+                                 const cv::Mat &camera_frame, int *numFaces,
                                  cv::Point2f **multi_face_landmarks) {
-  detector->GetFaceLandmarks(multi_face_landmarks);
+  detector->ProcessFrame2D(camera_frame, numFaces, multi_face_landmarks);
-}
 }
 
-const char MPFaceMeshDetector::kInputStream[] = "input_video";
+DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = MPFaceMeshDetector::kLandmarksNum;
-const char MPFaceMeshDetector::kOutputStream_landmarks[] =
+}
-    "multi_face_landmarks";
-const char MPFaceMeshDetector::kOutputStream_faceCount[] = "face_count";
 
 const std::string MPFaceMeshDetector::graphConfig = R"pb(
 # MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
@@ -197,7 +181,7 @@ output_stream: "face_count"
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"
-  input_stream: "FINISHED:multi_face_landmarks"
+  input_stream: "FINISHED:face_count"
   input_stream_info: {
     tag_index: "FINISHED"
     back_edge: true

mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h

@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <memory>
 #include <string>
+#include <windows.h>
 
 #include "absl/flags/flag.h"
 #include "absl/flags/parse.h"
@@ -29,24 +30,25 @@
 
 class MPFaceMeshDetector {
 public:
+  static constexpr auto kLandmarksNum = 468;
+
   MPFaceMeshDetector(int numFaces, const char *face_detection_model_path,
                      const char *face_landmark_model_path);
-  int GetFaceCount(const cv::Mat &camera_frame);
+
-  void GetFaceLandmarks(cv::Point2f **multi_face_landmarks);
+  void ProcessFrame2D(const cv::Mat &camera_frame, int *numFaces,
+                      cv::Point2f **multi_face_landmarks);
 
 private:
   absl::Status InitFaceMeshDetector(int numFaces,
                                     const char *face_detection_model_path,
                                     const char *face_landmark_model_path);
-  absl::Status ProcessFrameWithStatus(
+  absl::Status ProcessFrame2DWithStatus(const cv::Mat &camera_frame,
-      const cv::Mat &camera_frame,
+                                        int *numFaces,
-      std::vector<std::vector<cv::Point2f>> &multi_face_landmarks);
+                                        cv::Point2f **multi_face_landmarks);
-  absl::Status GetFaceCountWithStatus(const cv::Mat &camera_frame);
-  absl::Status GetFaceLandmarksWithStatus(cv::Point2f **multi_face_landmarks);
 
-  static const char kInputStream[];
+  static constexpr auto kInputStream = "input_video";
-  static const char kOutputStream_landmarks[];
+  static constexpr auto kOutputStream_landmarks = "multi_face_landmarks";
-  static const char kOutputStream_faceCount[];
+  static constexpr auto kOutputStream_faceCount = "face_count";
 
   static const std::string graphConfig;
 
@@ -54,30 +56,25 @@ private:
 
   std::unique_ptr<mediapipe::OutputStreamPoller> landmarks_poller_ptr;
   std::unique_ptr<mediapipe::OutputStreamPoller> face_count_poller_ptr;
-
-  int faceCount = -1;
 };
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct(
+DLLEXPORT MPFaceMeshDetector *
-    int numFaces = 1,
+MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path,
-    const char *face_detection_model_path =
+                            const char *face_landmark_model_path);
-        "mediapipe/modules/face_detection/face_detection_short_range.tflite",
-    const char *face_landmark_model_path =
-        "mediapipe/modules/face_landmark/face_landmark.tflite");
 
+DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector);
 
-DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector);
-
-DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector,
-                                            const cv::Mat &camera_frame);
 DLLEXPORT void
-FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector,
+MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector,
+                                 const cv::Mat &camera_frame, int *numFaces,
                                  cv::Point2f **multi_face_landmarks);
 
+DLLEXPORT extern const int MPFaceMeshDetectorLandmarksNum;
+
 #ifdef __cplusplus
 };
 #endif