Introduce AlignHandToPoseInWorldCalculator
PiperOrigin-RevId: 572959145
This commit is contained in:
parent
ac2d5cedbd
commit
2a286cc790
|
@ -35,6 +35,32 @@ cc_library(
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mediapipe_proto_library(
|
||||||
|
name = "align_hand_to_pose_in_world_calculator_proto",
|
||||||
|
srcs = ["align_hand_to_pose_in_world_calculator.proto"],
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/framework:calculator_options_proto",
|
||||||
|
"//mediapipe/framework:calculator_proto",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "align_hand_to_pose_in_world_calculator",
|
||||||
|
srcs = ["align_hand_to_pose_in_world_calculator.cc"],
|
||||||
|
hdrs = ["align_hand_to_pose_in_world_calculator.h"],
|
||||||
|
deps = [
|
||||||
|
":align_hand_to_pose_in_world_calculator_cc_proto",
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework/api2:node",
|
||||||
|
"//mediapipe/framework/api2:port",
|
||||||
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
|
"//mediapipe/framework/port:ret_check",
|
||||||
|
"//mediapipe/framework/port:status",
|
||||||
|
"@com_google_absl//absl/status",
|
||||||
|
],
|
||||||
|
alwayslink = 1,
|
||||||
|
)
|
||||||
|
|
||||||
mediapipe_proto_library(
|
mediapipe_proto_library(
|
||||||
name = "annotation_overlay_calculator_proto",
|
name = "annotation_overlay_calculator_proto",
|
||||||
srcs = ["annotation_overlay_calculator.proto"],
|
srcs = ["annotation_overlay_calculator.proto"],
|
||||||
|
|
|
@ -0,0 +1,80 @@
|
||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h"
|
||||||
|
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
#include "absl/status/status.h"
|
||||||
|
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
|
||||||
|
#include "mediapipe/framework/api2/node.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
|
||||||
|
namespace mediapipe::api2 {
|
||||||
|
|
||||||
|
namespace {} // namespace
|
||||||
|
|
||||||
|
class AlignHandToPoseInWorldCalculatorImpl
|
||||||
|
: public NodeImpl<AlignHandToPoseInWorldCalculator> {
|
||||||
|
public:
|
||||||
|
absl::Status Open(CalculatorContext* cc) override {
|
||||||
|
const auto& options =
|
||||||
|
cc->Options<mediapipe::AlignHandToPoseInWorldCalculatorOptions>();
|
||||||
|
hand_wrist_idx_ = options.hand_wrist_idx();
|
||||||
|
pose_wrist_idx_ = options.pose_wrist_idx();
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status Process(CalculatorContext* cc) override {
|
||||||
|
// Check that landmarks are not empty.
|
||||||
|
if (kInHandLandmarks(cc).IsEmpty()) {
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that pose landmarks are provided.
|
||||||
|
RET_CHECK(!kInPoseLandmarks(cc).IsEmpty());
|
||||||
|
|
||||||
|
const auto& in_hand_landmarks = kInHandLandmarks(cc).Get();
|
||||||
|
const auto& in_pose_landmarks = kInPoseLandmarks(cc).Get();
|
||||||
|
|
||||||
|
// Get hand and pose wrists.
|
||||||
|
RET_CHECK(hand_wrist_idx_ <= in_hand_landmarks.landmark_size());
|
||||||
|
const auto& hand_wrist = in_hand_landmarks.landmark(hand_wrist_idx_);
|
||||||
|
RET_CHECK(pose_wrist_idx_ <= in_pose_landmarks.landmark_size());
|
||||||
|
const auto& pose_wrist = in_pose_landmarks.landmark(pose_wrist_idx_);
|
||||||
|
|
||||||
|
LandmarkList out_hand_landmarks;
|
||||||
|
for (int i = 0; i < in_hand_landmarks.landmark_size(); ++i) {
|
||||||
|
const auto& in_landmark = in_hand_landmarks.landmark(i);
|
||||||
|
Landmark* out_landmark = out_hand_landmarks.add_landmark();
|
||||||
|
*out_landmark = in_landmark;
|
||||||
|
out_landmark->set_x(in_landmark.x() - hand_wrist.x() + pose_wrist.x());
|
||||||
|
out_landmark->set_y(in_landmark.y() - hand_wrist.y() + pose_wrist.y());
|
||||||
|
out_landmark->set_z(in_landmark.z() - hand_wrist.z() + pose_wrist.z());
|
||||||
|
}
|
||||||
|
|
||||||
|
kOutHandLandmarks(cc).Send(std::move(out_hand_landmarks));
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
int hand_wrist_idx_;
|
||||||
|
int pose_wrist_idx_;
|
||||||
|
};
|
||||||
|
MEDIAPIPE_NODE_IMPLEMENTATION(AlignHandToPoseInWorldCalculatorImpl);
|
||||||
|
|
||||||
|
} // namespace mediapipe::api2
|
|
@ -0,0 +1,74 @@
|
||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#ifndef MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
|
||||||
|
#define MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
|
||||||
|
|
||||||
|
#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h"
|
||||||
|
#include "mediapipe/framework/api2/node.h"
|
||||||
|
#include "mediapipe/framework/api2/port.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
|
||||||
|
namespace mediapipe::api2 {
|
||||||
|
|
||||||
|
// A calculator to align hand world landmarks with pose world landmarks.
|
||||||
|
//
|
||||||
|
// When `mediapipe.aimatter.LandmarksDetector` projects world landmarks from ROI
|
||||||
|
// local coordinates to original scene coordinates, it applies only rotation
|
||||||
|
// (derived from ROI) but neither scale nor translation. This calculator
|
||||||
|
// utilizes pose semantic to compensate this lack of information:
|
||||||
|
// - Translation is determined by matching wrist from hand landmarks with
|
||||||
|
// wrist from pose landmarks.
|
||||||
|
// - Scale can be determined (but is not at the moment) by calculating
|
||||||
|
// expected hand size from pose landmarks proportions.
|
||||||
|
//
|
||||||
|
// Input:
|
||||||
|
// HAND_LANDMARKS - LandmarkList
|
||||||
|
// Hand world landmarks.
|
||||||
|
// POSE_LANDMARKS - LandmarkList
|
||||||
|
// Pose world landmarks.
|
||||||
|
//
|
||||||
|
// Output:
|
||||||
|
// HAND_LANDMARKS - LandmarkList
|
||||||
|
// Aligned hand world landmarks.
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
// node {
|
||||||
|
// calculator: "AlignHandToPoseInWorldCalculator"
|
||||||
|
// input_stream: "HAND_LANDMARKS:hand_world_landmarks"
|
||||||
|
// input_stream: "POSE_LANDMARKS:pose_world_landmarks"
|
||||||
|
// output_stream: "HAND_LANDMARKS:hand_world_landmarks"
|
||||||
|
// options: {
|
||||||
|
// [mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
|
||||||
|
// hand_wrist_idx: 0
|
||||||
|
// pose_wrist_idx: 15 # 16 for right
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
class AlignHandToPoseInWorldCalculator : public NodeIntf {
|
||||||
|
public:
|
||||||
|
static constexpr Input<mediapipe::LandmarkList> kInHandLandmarks{
|
||||||
|
"HAND_LANDMARKS"};
|
||||||
|
static constexpr Input<mediapipe::LandmarkList> kInPoseLandmarks{
|
||||||
|
"POSE_LANDMARKS"};
|
||||||
|
static constexpr Output<mediapipe::LandmarkList> kOutHandLandmarks{
|
||||||
|
"HAND_LANDMARKS"};
|
||||||
|
MEDIAPIPE_NODE_INTERFACE(::mediapipe::AlignHandToPoseInWorldCalculator,
|
||||||
|
kInHandLandmarks, kInPoseLandmarks,
|
||||||
|
kOutHandLandmarks);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace mediapipe::api2
|
||||||
|
|
||||||
|
#endif // MEDIAPIPE_CALCULATORS_UTIL_ALIGN_HAND_TO_POSE_IN_WORLD_CALCULATOR_H_
|
|
@ -0,0 +1,31 @@
|
||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
syntax = "proto2";
|
||||||
|
|
||||||
|
package mediapipe;
|
||||||
|
|
||||||
|
import "mediapipe/framework/calculator.proto";
|
||||||
|
|
||||||
|
message AlignHandToPoseInWorldCalculatorOptions {
|
||||||
|
extend CalculatorOptions {
|
||||||
|
optional AlignHandToPoseInWorldCalculatorOptions ext = 403449802;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hand wrist index.
|
||||||
|
optional int32 hand_wrist_idx = 1;
|
||||||
|
|
||||||
|
// Pose wrist index.
|
||||||
|
optional int32 pose_wrist_idx = 2;
|
||||||
|
}
|
|
@ -5,7 +5,6 @@ rewrite_target_list = [
|
||||||
"adaptive_crop_calculator_proto",
|
"adaptive_crop_calculator_proto",
|
||||||
"affine_transform_data_proto",
|
"affine_transform_data_proto",
|
||||||
"affine_transform_from_rect_calculator_proto",
|
"affine_transform_from_rect_calculator_proto",
|
||||||
"align_hand_to_pose_in_world_calculator_proto",
|
|
||||||
"anchor_proto",
|
"anchor_proto",
|
||||||
"annotation_overlay_calculator_proto",
|
"annotation_overlay_calculator_proto",
|
||||||
"annotation_proto",
|
"annotation_proto",
|
||||||
|
|
|
@ -165,12 +165,12 @@ node {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
node {
|
node {
|
||||||
calculator: "mediapipe.aimatter.AlignHandToPoseInWorldCalculator"
|
calculator: "AlignHandToPoseInWorldCalculator"
|
||||||
input_stream: "HAND_LANDMARKS:__stream_14"
|
input_stream: "HAND_LANDMARKS:__stream_14"
|
||||||
input_stream: "POSE_LANDMARKS:pose_world_landmarks_in"
|
input_stream: "POSE_LANDMARKS:pose_world_landmarks_in"
|
||||||
output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out"
|
output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out"
|
||||||
options {
|
options {
|
||||||
[mediapipe.aimatter.AlignHandToPoseInWorldCalculatorOptions.ext] {
|
[mediapipe.AlignHandToPoseInWorldCalculatorOptions.ext] {
|
||||||
hand_wrist_idx: 0
|
hand_wrist_idx: 0
|
||||||
pose_wrist_idx: 15
|
pose_wrist_idx: 15
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user