feat: added ouputing of multi-face_landmarks through dll to c++

Change List:
- now FaceMeshDetector::ProcessFrame2D returns pointer to vector with all facelandmars
- added converting of `mediapipe::NormalizedLandmarkList` to `std::vector<cv::Point2f>` to make more independent of mediapipe (to use in custom projects)

mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp

@@ -4,7 +4,6 @@ int main(int argc, char **argv) {
   google::InitGoogleLogging(argv[0]);
   absl::ParseCommandLine(argc, argv);
 
-
   cv::VideoCapture capture;
   capture.open(0);
   if (!capture.isOpened()) {
@@ -41,7 +40,21 @@ int main(int argc, char **argv) {
     cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
     cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
 
-    FaceMeshDetector_ProcessFrame(faceMeshDetector, camera_frame);
+    std::unique_ptr<std::vector<std::vector<cv::Point2f>>> multi_face_landmarks(
+        reinterpret_cast<std::vector<std::vector<cv::Point2f>> *>(
+            FaceMeshDetector_ProcessFrame2D(faceMeshDetector, camera_frame)));
+
+    const auto multi_face_landmarks_num = multi_face_landmarks->size();
+
+    LOG(INFO) << "Got multi_face_landmarks_num: " << multi_face_landmarks_num;
+
+    if (multi_face_landmarks_num) {
+      auto &face_landmarks = multi_face_landmarks->operator[](0);
+      auto &landmark = face_landmarks[0];
+
+      LOG(INFO) << "First landmark: x - " << landmark.x << ", y - "
+                << landmark.y;
+    }
 
     const int pressed_key = cv::waitKey(5);
     if (pressed_key >= 0 && pressed_key != 255)

mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp

@@ -2,8 +2,6 @@
 
 #include "face_mesh_lib.h"
 
-#define DEBUG
-
 FaceMeshDetector::FaceMeshDetector() {
   const auto status = InitFaceMeshDetector();
   if (!status.ok()) {
@@ -39,7 +37,10 @@ absl::Status FaceMeshDetector::InitFaceMeshDetector() {
   return absl::Status();
 }
 
-absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) {
+absl::Status FaceMeshDetector::ProcessFrameWithStatus(
+    cv::Mat &camera_frame,
+    std::unique_ptr<std::vector<std::vector<cv::Point2f>>>
+        &multi_face_landmarks) {
   // Wrap Mat into an ImageFrame.
   auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
       mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
@@ -88,26 +89,38 @@ absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) {
       face_landmarks_packet
           .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>();
 
-  auto &output_landmarks = output_landmarks_vector[0];
+  multi_face_landmarks->reserve(output_landmarks_vector.size());
+
+  for (const auto &normalizedLandmarkList : output_landmarks_vector) {
+    multi_face_landmarks->emplace_back();
+
+    auto &face_landmarks = multi_face_landmarks->back();
+
+    const auto landmarks_num = normalizedLandmarkList.landmark_size();
 
 #ifdef DEBUG
-  LOG(INFO) << "Got landmarks_packet: " << output_landmarks.landmark_size();
+    LOG(INFO) << "Got landmarks_num: " << landmarks_num;
 #endif
 
-  auto &landmark = output_landmarks.landmark(0);
+    face_landmarks.reserve(landmarks_num);
-#ifdef DEBUG
+
-  LOG(INFO) << "First landmark: x - " << landmark.x() << ", y - "
+    for (int i = 0; i < landmarks_num; ++i) {
-            << landmark.y() << ", z - " << landmark.z();
+      auto &landmark = normalizedLandmarkList.landmark(i);
-#endif
+
+      face_landmarks.emplace_back(landmark.x(), landmark.y());
+    }
+  }
 
   return absl::Status();
 }
 
-std::vector<cv::Point2f> *
+std::vector<std::vector<cv::Point2f>> *
-FaceMeshDetector::ProcessFrame(cv::Mat &camera_frame) {
+FaceMeshDetector::ProcessFrame2D(cv::Mat &camera_frame) {
-  ProcessFrameWithStatus(camera_frame);
+  auto landmarks = std::make_unique<std::vector<std::vector<cv::Point2f>>>();
 
-  return new std::vector<cv::Point2f>();
+  ProcessFrameWithStatus(camera_frame, landmarks);
+
+  return landmarks.release();
 }
 
 extern "C" {
@@ -119,9 +132,9 @@ DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) {
   delete detector;
 }
 
-DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector,
+DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector,
-                                              cv::Mat &camera_frame) {
+                                                cv::Mat &camera_frame) {
-  return reinterpret_cast<void *>(detector->ProcessFrame(camera_frame));
+  return reinterpret_cast<void *>(detector->ProcessFrame2D(camera_frame));
 }
 }
 

mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h

@@ -8,8 +8,8 @@
 #endif
 
 #include <cstdlib>
-#include <string>
 #include <memory>
+#include <string>
 
 #include "absl/flags/flag.h"
 #include "absl/flags/parse.h"
@@ -29,11 +29,14 @@ class FaceMeshDetector {
 public:
   FaceMeshDetector();
   ~FaceMeshDetector() = default;
-  std::vector<cv::Point2f> *ProcessFrame(cv::Mat &camera_frame);
+  std::vector<std::vector<cv::Point2f>> *ProcessFrame2D(cv::Mat &camera_frame);
 
 private:
   absl::Status InitFaceMeshDetector();
-  absl::Status ProcessFrameWithStatus(cv::Mat &camera_frame);
+  absl::Status
+  ProcessFrameWithStatus(cv::Mat &camera_frame,
+                         std::unique_ptr<std::vector<std::vector<cv::Point2f>>>
+                             &multi_face_landmarks);
 
   static const char kInputStream[];
   static const char kOutputStream_landmarks[];
@@ -55,8 +58,8 @@ DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct();
 
 DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector);
 
-DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector,
+DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector,
-                                              cv::Mat &camera_frame);
+                                                cv::Mat &camera_frame);
 
 #ifdef __cplusplus
 };