From 3dd64807050d4eca6fb660503a2f38aa194977cf Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 9 Oct 2023 13:53:51 -0700 Subject: [PATCH] No public description PiperOrigin-RevId: 572032324 --- .../tasks/cc/vision/hand_landmarker/BUILD | 1 + .../hand_landmarks_detector_graph.cc | 14 +- ...listic_hand_tracking_left_hand_graph.pbtxt | 183 ++++++++++++++++++ 3 files changed, 188 insertions(+), 10 deletions(-) create mode 100644 mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD index 1e24256d1..5449810ef 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD @@ -85,6 +85,7 @@ cc_library( "//mediapipe/calculators/util:world_landmark_projection_calculator", "//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:status", "//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:landmark_cc_proto", diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc index 1d4e1b2c1..38f639d2c 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc @@ -13,8 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include -#include #include #include @@ -31,20 +29,16 @@ limitations under the License. #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/components/processors/image_preprocessing_graph.h" #include "mediapipe/tasks/cc/components/utils/gate.h" #include "mediapipe/tasks/cc/core/model_resources.h" #include "mediapipe/tasks/cc/core/model_task_graph.h" #include "mediapipe/tasks/cc/core/proto/inference_subgraph.pb.h" -#include "mediapipe/tasks/cc/core/utils.h" -#include "mediapipe/tasks/cc/metadata/metadata_extractor.h" #include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/utils/image_tensor_specs.h" -#include "mediapipe/tasks/metadata/metadata_schema_generated.h" #include "mediapipe/util/label_map.pb.h" -#include "mediapipe/util/label_map_util.h" namespace mediapipe { namespace tasks { @@ -214,7 +208,7 @@ class SingleHandLandmarksDetectorGraph : public core::ModelTaskGraph { SubgraphContext* sc) override { MP_ASSIGN_OR_RETURN( const auto* model_resources, - CreateModelResources(sc)); + GetOrCreateModelResources(sc)); Graph graph; MP_ASSIGN_OR_RETURN( auto hand_landmark_detection_outs, @@ -492,8 +486,8 @@ class MultipleHandLandmarksDetectorGraph : public core::ModelTaskGraph { auto& hand_landmark_subgraph = graph.AddNode( "mediapipe.tasks.vision.hand_landmarker." "SingleHandLandmarksDetectorGraph"); - hand_landmark_subgraph.GetOptions() - .CopyFrom(subgraph_options); + hand_landmark_subgraph.GetOptions() = + subgraph_options; auto& begin_loop_multi_hand_rects = graph.AddNode("BeginLoopNormalizedRectCalculator"); diff --git a/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt b/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt new file mode 100644 index 000000000..242a092c1 --- /dev/null +++ b/mediapipe/tasks/testdata/vision/holistic_hand_tracking_left_hand_graph.pbtxt @@ -0,0 +1,183 @@ +node { + calculator: "SplitNormalizedLandmarkListCalculator" + input_stream: "pose_landmarks_in" + output_stream: "__stream_0" + options { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges { + begin: 15 + end: 16 + } + ranges { + begin: 17 + end: 18 + } + ranges { + begin: 19 + end: 20 + } + combine_outputs: true + } + } +} +node { + calculator: "SplitNormalizedLandmarkListCalculator" + input_stream: "__stream_0" + output_stream: "__stream_1" + options { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges { + begin: 0 + end: 1 + } + combine_outputs: true + } + } +} +node { + calculator: "LandmarkVisibilityCalculator" + input_stream: "NORM_LANDMARKS:__stream_1" + output_stream: "VISIBILITY:__stream_2" +} +node { + calculator: "ThresholdingCalculator" + input_stream: "FLOAT:__stream_2" + output_stream: "FLAG:__stream_3" + options { + [mediapipe.ThresholdingCalculatorOptions.ext] { + threshold: 0.1 + } + } +} +node { + calculator: "GateCalculator" + input_stream: "__stream_0" + input_stream: "ALLOW:__stream_3" + output_stream: "__stream_4" +} +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image_in" + output_stream: "SIZE:__stream_5" +} +node { + calculator: "LandmarksToDetectionCalculator" + input_stream: "NORM_LANDMARKS:__stream_4" + output_stream: "DETECTION:__stream_6" +} +node { + calculator: "HandDetectionsFromPoseToRectsCalculator" + input_stream: "DETECTION:__stream_6" + input_stream: "IMAGE_SIZE:__stream_5" + output_stream: "NORM_RECT:__stream_7" +} +node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_5" + input_stream: "NORM_RECT:__stream_7" + output_stream: "__stream_8" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2.7 + scale_y: 2.7 + shift_x: 0 + shift_y: -0.1 + square_long: true + } + } +} +node { + calculator: "mediapipe.tasks.vision.hand_landmarker.HandRoiRefinementGraph" + input_stream: "IMAGE:image_in" + input_stream: "NORM_RECT:__stream_8" + output_stream: "NORM_RECT:__stream_9" + node_options { + [type.googleapis.com/mediapipe.tasks.vision.hand_landmarker.proto.HandRoiRefinementGraphOptions] { + base_options { + model_asset { + file_name: "$0/mediapipe/tasks/testdata/vision/handrecrop_2020_07_21_v0.f16.tflite" + } + } + } + } +} +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "LOOP:left_hand_landmarks_out" + input_stream: "MAIN:__stream_5" + output_stream: "PREV_LOOP:__stream_10" +} +node { + calculator: "HandLandmarksToRectCalculator" + input_stream: "IMAGE_SIZE:__stream_5" + input_stream: "NORM_LANDMARKS:__stream_10" + output_stream: "NORM_RECT:__stream_11" +} +node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_5" + input_stream: "NORM_RECT:__stream_11" + output_stream: "__stream_12" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 2 + shift_x: 0 + shift_y: -0.1 + square_long: true + } + } +} +node { + calculator: "RoiTrackingCalculator" + input_stream: "IMAGE_SIZE:__stream_5" + input_stream: "PREV_LANDMARKS:__stream_10" + input_stream: "PREV_LANDMARKS_RECT:__stream_12" + input_stream: "RECROP_RECT:__stream_9" + output_stream: "TRACKING_RECT:__stream_13" + options { + [mediapipe.RoiTrackingCalculatorOptions.ext] { + rect_requirements { + rotation_degrees: 40 + translation: 0.2 + scale: 0.4 + } + landmarks_requirements { + recrop_rect_margin: -0.1 + } + } + } +} +node { + calculator: "mediapipe.tasks.vision.hand_landmarker.SingleHandLandmarksDetectorGraph" + input_stream: "HAND_RECT:__stream_13" + input_stream: "IMAGE:image_in" + output_stream: "LANDMARKS:left_hand_landmarks_out" + output_stream: "WORLD_LANDMARKS:__stream_14" + options { + [mediapipe.tasks.vision.hand_landmarker.proto.HandLandmarksDetectorGraphOptions.ext] { + base_options { + model_asset { + file_name: "$1/mediapipe/tasks/testdata/vision/hand_landmark_full.tflite" + } + } + } + } +} +node { + calculator: "mediapipe.aimatter.AlignHandToPoseInWorldCalculator" + input_stream: "HAND_LANDMARKS:__stream_14" + input_stream: "POSE_LANDMARKS:pose_world_landmarks_in" + output_stream: "HAND_LANDMARKS:left_hand_world_landmarks_out" + options { + [mediapipe.aimatter.AlignHandToPoseInWorldCalculatorOptions.ext] { + hand_wrist_idx: 0 + pose_wrist_idx: 15 + } + } +} +input_stream: "IMAGE:image_in" +input_stream: "POSE_LANDMARKS:pose_landmarks_in" +input_stream: "POSE_WORLD_LANDMARKS:pose_world_landmarks_in" +output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks_out" +output_stream: "LEFT_HAND_WORLD_LANDMARKS:left_hand_world_landmarks_out"