Introduce AlignHandToPoseInWorldCalculator

PiperOrigin-RevId: 572959145

mediapipe/calculators/util/BUILD

@@ -35,6 +35,32 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "align_hand_to_pose_in_world_calculator_proto",
+    srcs = ["align_hand_to_pose_in_world_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+cc_library(
+    name = "align_hand_to_pose_in_world_calculator",
+    srcs = ["align_hand_to_pose_in_world_calculator.cc"],
+    hdrs = ["align_hand_to_pose_in_world_calculator.h"],
+    deps = [
+        ":align_hand_to_pose_in_world_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/api2:port",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+    ],
+    alwayslink = 1,
+)
+
 mediapipe_proto_library(
     name = "annotation_overlay_calculator_proto",
     srcs = ["annotation_overlay_calculator.proto"],

mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.cc

@@ -0,0 +1,80 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h"
+
+#include <utility>
+
+#include "absl/status/status.h"
+#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe::api2 {
+
+namespace {}  // namespace
+
+class AlignHandToPoseInWorldCalculatorImpl
+    : public NodeImpl<AlignHandToPoseInWorldCalculator> {
+ public:
+  absl::Status Open(CalculatorContext* cc) override {
+    const auto& options =
+        cc->Options<mediapipe::AlignHandToPoseInWorldCalculatorOptions>();
+    hand_wrist_idx_ = options.hand_wrist_idx();
+    pose_wrist_idx_ = options.pose_wrist_idx();
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+    // Check that landmarks are not empty.
+    if (kInHandLandmarks(cc).IsEmpty()) {
+      return absl::OkStatus();
+    }
+
+    // Check that pose landmarks are provided.
+    RET_CHECK(!kInPoseLandmarks(cc).IsEmpty());
+
+    const auto& in_hand_landmarks = kInHandLandmarks(cc).Get();
+    const auto& in_pose_landmarks = kInPoseLandmarks(cc).Get();
+
+    // Get hand and pose wrists.
+    RET_CHECK(hand_wrist_idx_ <= in_hand_landmarks.landmark_size());
+    const auto& hand_wrist = in_hand_landmarks.landmark(hand_wrist_idx_);
+    RET_CHECK(pose_wrist_idx_ <= in_pose_landmarks.landmark_size());
+    const auto& pose_wrist = in_pose_landmarks.landmark(pose_wrist_idx_);
+
+    LandmarkList out_hand_landmarks;
+    for (int i = 0; i < in_hand_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_hand_landmarks.landmark(i);
+      Landmark* out_landmark = out_hand_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(in_landmark.x() - hand_wrist.x() + pose_wrist.x());
+      out_landmark->set_y(in_landmark.y() - hand_wrist.y() + pose_wrist.y());
+      out_landmark->set_z(in_landmark.z() - hand_wrist.z() + pose_wrist.z());
+    }
+
+    kOutHandLandmarks(cc).Send(std::move(out_hand_landmarks));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  int hand_wrist_idx_;
+  int pose_wrist_idx_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(AlignHandToPoseInWorldCalculatorImpl);
+
+}  // namespace mediapipe::api2

mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h

@@ -0,0 +1,74 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
+
+#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe::api2 {
+
+// A calculator to align hand world landmarks with pose world landmarks.
+//
+// When `mediapipe.aimatter.LandmarksDetector` projects world landmarks from ROI
+// local coordinates to original scene coordinates, it applies only rotation
+// (derived from ROI) but neither scale nor translation. This calculator
+// utilizes pose semantic to compensate this lack of information:
+//   - Translation is determined by matching wrist from hand landmarks with
+//     wrist from pose landmarks.
+//   - Scale can be determined (but is not at the moment) by calculating
+//     expected hand size from pose landmarks proportions.
+//
+// Input:
+//   HAND_LANDMARKS - LandmarkList
+//     Hand world landmarks.
+//   POSE_LANDMARKS - LandmarkList
+//     Pose world landmarks.
+//
+// Output:
+//   HAND_LANDMARKS - LandmarkList
+//     Aligned hand world landmarks.
+//
+// Example:
+//   node {
+//     calculator: "AlignHandToPoseInWorldCalculator"
+//     input_stream: "HAND_LANDMARKS:hand_world_landmarks"
+//     input_stream: "POSE_LANDMARKS:pose_world_landmarks"
+//     output_stream: "HAND_LANDMARKS:hand_world_landmarks"
+//     options: {
+//       [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
+//         hand_wrist_idx: 0
+//         pose_wrist_idx: 15    # 16 for right
+//       }
+//     }
+//   }
+class AlignHandToPoseInWorldCalculator : public NodeIntf {
+ public:
+  static constexpr Input<mediapipe::LandmarkList> kInHandLandmarks{
+      "HAND_LANDMARKS"};
+  static constexpr Input<mediapipe::LandmarkList> kInPoseLandmarks{
+      "POSE_LANDMARKS"};
+  static constexpr Output<mediapipe::LandmarkList> kOutHandLandmarks{
+      "HAND_LANDMARKS"};
+  MEDIAPIPE_NODE_INTERFACE(::mediapipe::AlignHandToPoseInWorldCalculator,
+                           kInHandLandmarks, kInPoseLandmarks,
+                           kOutHandLandmarks);
+};
+
+}  // namespace mediapipe::api2
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_

mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message AlignHandToPoseInWorldCalculatorOptions {
+  extend CalculatorOptions {
+    optional AlignHandToPoseInWorldCalculatorOptions ext = 403449802;
+  }
+
+  // Hand wrist index.
+  optional int32 hand_wrist_idx = 1;
+
+  // Pose wrist index.
+  optional int32 pose_wrist_idx = 2;
+}

mediapipe/framework/tool/mediapipe_proto_allowlist.bzl

@@ -5,7 +5,6 @@ rewrite_target_list = [
     "adaptive_crop_calculator_proto",
     "affine_transform_data_proto",
     "affine_transform_from_rect_calculator_proto",
-    "align_hand_to_pose_in_world_calculator_proto",
     "anchor_proto",
     "annotation_overlay_calculator_proto",
     "annotation_proto",

mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt

@@ -165,12 +165,12 @@ node {
   }
 }
 node {
-  calculator: "mediapipe.aimatter.AlignHandToPoseInWorldCalculator"
+  calculator: "AlignHandToPoseInWorldCalculator"
   input_stream: "HAND_LANDMARKS:__stream_14"
   input_stream: "POSE_LANDMARKS:pose_world_landmarks_in"
   output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out"
   options {
-    [mediapipe.aimatter.AlignHandToPoseInWorldCalculatorOptions.ext] {
+    [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
       hand_wrist_idx: 0
       pose_wrist_idx: 15
     }