cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce AlignHandToPoseInWorldCalculator

Browse Source 
PiperOrigin-RevId: 572959145
This commit is contained in:
MediaPipe Team 2023-10-12 11:09:19 -07:00 committed by Copybara-Service
parent ac2d5cedbd
commit 2a286cc790
6 changed files with 213 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
mediapipe/calculators/util/BUILD
View File

@ -35,6 +35,32 @@ cc_library(
alwayslink = 1,
)
mediapipe_proto_library(
name = "align_hand_to_pose_in_world_calculator_proto",
srcs = ["align_hand_to_pose_in_world_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "align_hand_to_pose_in_world_calculator",
srcs = ["align_hand_to_pose_in_world_calculator.cc"],
hdrs = ["align_hand_to_pose_in_world_calculator.h"],
deps = [
":align_hand_to_pose_in_world_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "annotation_overlay_calculator_proto",
srcs = ["annotation_overlay_calculator.proto"],

80
mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.cc Normal file
View File

@ -0,0 +1,80 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h"
#include <utility>
#include "absl/status/status.h"
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe::api2 {
namespace {} // namespace
class AlignHandToPoseInWorldCalculatorImpl
: public NodeImpl<AlignHandToPoseInWorldCalculator> {
public:
absl::Status Open(CalculatorContext* cc) override {
const auto& options =
cc->Options<mediapipe::AlignHandToPoseInWorldCalculatorOptions>();
hand_wrist_idx_ = options.hand_wrist_idx();
pose_wrist_idx_ = options.pose_wrist_idx();
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
// Check that landmarks are not empty.
if (kInHandLandmarks(cc).IsEmpty()) {
return absl::OkStatus();
}
// Check that pose landmarks are provided.
RET_CHECK(!kInPoseLandmarks(cc).IsEmpty());
const auto& in_hand_landmarks = kInHandLandmarks(cc).Get();
const auto& in_pose_landmarks = kInPoseLandmarks(cc).Get();
// Get hand and pose wrists.
RET_CHECK(hand_wrist_idx_ <= in_hand_landmarks.landmark_size());
const auto& hand_wrist = in_hand_landmarks.landmark(hand_wrist_idx_);
RET_CHECK(pose_wrist_idx_ <= in_pose_landmarks.landmark_size());
const auto& pose_wrist = in_pose_landmarks.landmark(pose_wrist_idx_);
LandmarkList out_hand_landmarks;
for (int i = 0; i < in_hand_landmarks.landmark_size(); ++i) {
const auto& in_landmark = in_hand_landmarks.landmark(i);
Landmark* out_landmark = out_hand_landmarks.add_landmark();
*out_landmark = in_landmark;
out_landmark->set_x(in_landmark.x() - hand_wrist.x() + pose_wrist.x());
out_landmark->set_y(in_landmark.y() - hand_wrist.y() + pose_wrist.y());
out_landmark->set_z(in_landmark.z() - hand_wrist.z() + pose_wrist.z());
}
kOutHandLandmarks(cc).Send(std::move(out_hand_landmarks));
return absl::OkStatus();
}
private:
int hand_wrist_idx_;
int pose_wrist_idx_;
};
MEDIAPIPE_NODE_IMPLEMENTATION(AlignHandToPoseInWorldCalculatorImpl);
} // namespace mediapipe::api2

74
mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h Normal file
View File

@ -0,0 +1,74 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/formats/landmark.pb.h"
namespace mediapipe::api2 {
// A calculator to align hand world landmarks with pose world landmarks.
//
// When `mediapipe.aimatter.LandmarksDetector` projects world landmarks from ROI
// local coordinates to original scene coordinates, it applies only rotation
// (derived from ROI) but neither scale nor translation. This calculator
// utilizes pose semantic to compensate this lack of information:
// - Translation is determined by matching wrist from hand landmarks with
// wrist from pose landmarks.
// - Scale can be determined (but is not at the moment) by calculating
// expected hand size from pose landmarks proportions.
//
// Input:
// HAND_LANDMARKS - LandmarkList
// Hand world landmarks.
// POSE_LANDMARKS - LandmarkList
// Pose world landmarks.
//
// Output:
// HAND_LANDMARKS - LandmarkList
// Aligned hand world landmarks.
//
// Example:
// node {
// calculator: "AlignHandToPoseInWorldCalculator"
// input_stream: "HAND_LANDMARKS:hand_world_landmarks"
// input_stream: "POSE_LANDMARKS:pose_world_landmarks"
// output_stream: "HAND_LANDMARKS:hand_world_landmarks"
// options: {
// [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
// hand_wrist_idx: 0
// pose_wrist_idx: 15 # 16 for right
// }
// }
// }
class AlignHandToPoseInWorldCalculator : public NodeIntf {
public:
static constexpr Input<mediapipe::LandmarkList> kInHandLandmarks{
"HAND_LANDMARKS"};
static constexpr Input<mediapipe::LandmarkList> kInPoseLandmarks{
"POSE_LANDMARKS"};
static constexpr Output<mediapipe::LandmarkList> kOutHandLandmarks{
"HAND_LANDMARKS"};
MEDIAPIPE_NODE_INTERFACE(::mediapipe::AlignHandToPoseInWorldCalculator,
kInHandLandmarks, kInPoseLandmarks,
kOutHandLandmarks);
};
} // namespace mediapipe::api2
#endif // MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_

31
mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.proto Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message AlignHandToPoseInWorldCalculatorOptions {
extend CalculatorOptions {
optional AlignHandToPoseInWorldCalculatorOptions ext = 403449802;
}
// Hand wrist index.
optional int32 hand_wrist_idx = 1;
// Pose wrist index.
optional int32 pose_wrist_idx = 2;
}

1
mediapipe/framework/tool/mediapipe_proto_allowlist.bzl
View File

@ -5,7 +5,6 @@ rewrite_target_list = [
"adaptive_crop_calculator_proto",
"affine_transform_data_proto",
"affine_transform_from_rect_calculator_proto",
"align_hand_to_pose_in_world_calculator_proto",
"anchor_proto",
"annotation_overlay_calculator_proto",
"annotation_proto",

4
mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt vendored
View File

@ -165,12 +165,12 @@ node {
}
}
node {
calculator: "mediapipe.aimatter.AlignHandToPoseInWorldCalculator"
calculator: "AlignHandToPoseInWorldCalculator"
input_stream: "HAND_LANDMARKS:__stream_14"
input_stream: "POSE_LANDMARKS:pose_world_landmarks_in"
output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out"
options {
[mediapipe.aimatter.AlignHandToPoseInWorldCalculatorOptions.ext] {
[mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
hand_wrist_idx: 0
pose_wrist_idx: 15
}