Merge pull request #3 from NimagnaAG/feature/landmark-visibility

Add the possibility to get landmark visibility

mediapipe/pose_tracking_dll/README.md

@@ -77,6 +77,21 @@ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 --action_env PYTHON_BIN_PATH
 
 The results will be stored in the `bazel-bin\mediapipe\pose_tracking_dll` folder.
 
+In the case the build stalls, pressing Ctrl+C might not be sufficient to stop the task. In that case, if you try to (resume the) build again,
+the following message will be displayed:
+
+```
+Another command (pid=5300) is running. Waiting for it to complete on the server (server_pid=3684)
+```
+
+Unfortunately this process is hidden for some reason and can't be found in taskmgr. Fortunately, you can use the `taskkill` command to kill the process:
+
+```
+taskkill /F /PID 3684
+```
+
+After that, you should be able to run the build command again.
+
 ### Build debug symbols
 - `dbg` can be used in place of `opt` to build the library with debug symbols in Visual Studio pdb format.
 

mediapipe/pose_tracking_dll/pose_tracking.cpp

@@ -36,6 +36,7 @@ class PoseTrackingImpl {
  public:
   PoseTrackingImpl(const std::string& calculatorGraphConfigFile) {
     auto status = initialize(calculatorGraphConfigFile);
+	LOG(WARNING) << "Initialized PoseTracking with status: " << status;
   }
 
   absl::Status initialize(const std::string& calculatorGraphConfigFile) {
@@ -116,6 +117,7 @@ class PoseTrackingImpl {
       poseLandmarks[j].x = landmark.x();
       poseLandmarks[j].y = landmark.y();
       poseLandmarks[j].z = landmark.z();
+      visibility[j] = landmark.visibility();
     }
 
     return absl::OkStatus();
@@ -124,6 +126,7 @@ class PoseTrackingImpl {
   nimagna::cv_wrapper::Point3f* lastDetectedLandmarks() { return poseLandmarks; }
 
   cv::Mat lastSegmentedFrame() { return segmentedMask; }
+  float* landmarksVisibility() { return visibility; }
 
   static constexpr size_t kLandmarksCount = 33u;
 
@@ -131,6 +134,7 @@ class PoseTrackingImpl {
   mediapipe::Packet poseLandmarksPacket;
   cv::Mat segmentedMask;
   nimagna::cv_wrapper::Point3f poseLandmarks[kLandmarksCount];
+  float visibility[kLandmarksCount] = {0};
   std::unique_ptr<mediapipe::OutputStreamPoller> maskPollerPtr;
   std::unique_ptr<mediapipe::OutputStreamPoller> landmarksPollerPtr;
   mediapipe::CalculatorGraph graph;
@@ -149,8 +153,8 @@ bool PoseTracking::processFrame(const cv_wrapper::Mat& inputRGB8Bit) {
   return mImplementation->processFrame(frame);
 }
 
-cv_wrapper::Point3f* PoseTracking::lastDetectedLandmarks() {
-  return mImplementation->lastDetectedLandmarks();
+PoseTracking::PoseLandmarks PoseTracking::lastDetectedLandmarks() {
+  return {mImplementation->lastDetectedLandmarks(), mImplementation->landmarksVisibility()};
 }
 
 cv_wrapper::Mat PoseTracking::lastSegmentedFrame() {
@@ -163,5 +167,4 @@ PoseTracking::~PoseTracking()
 {
 	delete mImplementation;
 }
-
 }  // namespace nimagna

mediapipe/pose_tracking_dll/pose_tracking.h

@@ -62,9 +62,16 @@ struct Mat {
 };
 }  // namespace cv_wrapper
 
+  
 class DLLEXPORT PoseTracking {
  public:
-  static constexpr size_t kLandmarksCount = 33u;
+  struct PoseLandmarks {
+    PoseLandmarks(cv_wrapper::Point3f* points, float* visibility) : points(points), visibility(visibility) {}
+    static constexpr size_t kLandmarksCount = 33u;
+    const cv_wrapper::Point3f* points;
+    const float* visibility;
+  };
+
   enum LandmarkNames {
     NOSE = 0,
     LEFT_EYE_INNER,
@@ -99,7 +106,7 @@ class DLLEXPORT PoseTracking {
     RIGHT_HEEL,
     LEFT_FOOT_INDEX,
     RIGHT_FOOT_INDEX,
-    COUNT = kLandmarksCount
+    COUNT = PoseLandmarks::kLandmarksCount
   };
 
   PoseTracking(const char* calculatorGraphConfigFile);
@@ -107,7 +114,7 @@ class DLLEXPORT PoseTracking {
 
   bool processFrame(const cv_wrapper::Mat& inputRGB8Bit);
   cv_wrapper::Mat lastSegmentedFrame();
-  cv_wrapper::Point3f* lastDetectedLandmarks();
+  PoseTracking::PoseLandmarks lastDetectedLandmarks();
 
  private:
   PoseTrackingImpl* mImplementation;