From 06cc6d1546788fbdedd0babe41a51d3fa8947d6e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 18 Oct 2023 10:56:07 -0700 Subject: [PATCH] No public description PiperOrigin-RevId: 574528013 --- mediapipe/tasks/cc/vision/pose_detector/BUILD | 1 + .../pose_landmarks_connections.h | 42 +++- .../vision/holistic_pose_tracking_graph.pbtxt | 238 ++++++++++++++++++ 3 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 mediapipe/tasks/testdata/vision/holistic_pose_tracking_graph.pbtxt diff --git a/mediapipe/tasks/cc/vision/pose_detector/BUILD b/mediapipe/tasks/cc/vision/pose_detector/BUILD index 4f9bc5944..d1998a716 100644 --- a/mediapipe/tasks/cc/vision/pose_detector/BUILD +++ b/mediapipe/tasks/cc/vision/pose_detector/BUILD @@ -22,6 +22,7 @@ cc_library( name = "pose_detector_graph", srcs = ["pose_detector_graph.cc"], deps = [ + "//mediapipe/calculators/core:clip_vector_size_calculator", "//mediapipe/calculators/core:clip_vector_size_calculator_cc_proto", "//mediapipe/calculators/tensor:image_to_tensor_calculator_cc_proto", "//mediapipe/calculators/tensor:inference_calculator", diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h index 4b79215a4..1e5b400a1 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h @@ -23,12 +23,42 @@ namespace tasks { namespace vision { namespace pose_landmarker { -static constexpr std::array, 34> kPoseLandmarksConnections{{ - {1, 2}, {0, 1}, {2, 3}, {3, 7}, {0, 4}, {4, 5}, {5, 6}, - {6, 8}, {9, 10}, {11, 12}, {11, 13}, {13, 15}, {15, 17}, {15, 19}, - {15, 21}, {17, 19}, {12, 14}, {14, 16}, {16, 18}, {16, 20}, {16, 22}, - {18, 20}, {11, 23}, {12, 24}, {23, 24}, {23, 25}, {24, 26}, {25, 27}, - {26, 28}, {27, 29}, {28, 30}, {29, 31}, {30, 32}, {27, 31}, +static constexpr std::array, 35> kPoseLandmarksConnections{{ + {0, 4}, // (nose, right_eye_inner) + {4, 5}, // (right_eye_inner, right_eye) + {5, 6}, // (right_eye, right_eye_outer) + {6, 8}, // (right_eye_outer, right_ear) + {0, 1}, // (nose, left_eye_inner) + {1, 2}, // (left_eye_inner, left_eye) + {2, 3}, // (left_eye, left_eye_outer) + {3, 7}, // (left_eye_outer, left_ear) + {10, 9}, // (mouth_right, mouth_left) + {12, 11}, // (right_shoulder, left_shoulder) + {12, 14}, // (right_shoulder, right_elbow) + {14, 16}, // (right_elbow, right_wrist) + {16, 18}, // (right_wrist, right_pinky_1) + {16, 20}, // (right_wrist, right_index_1) + {16, 22}, // (right_wrist, right_thumb_2) + {18, 20}, // (right_pinky_1, right_index_1) + {11, 13}, // (left_shoulder, left_elbow) + {13, 15}, // (left_elbow, left_wrist) + {15, 17}, // (left_wrist, left_pinky_1) + {15, 19}, // (left_wrist, left_index_1) + {15, 21}, // (left_wrist, left_thumb_2) + {17, 19}, // (left_pinky_1, left_index_1) + {12, 24}, // (right_shoulder, right_hip) + {11, 23}, // (left_shoulder, left_hip) + {24, 23}, // (right_hip, left_hip) + {24, 26}, // (right_hip, right_knee) + {23, 25}, // (left_hip, left_knee) + {26, 28}, // (right_knee, right_ankle) + {25, 27}, // (left_knee, left_ankle) + {28, 30}, // (right_ankle, right_heel) + {27, 29}, // (left_ankle, left_heel) + {30, 32}, // (right_heel, right_foot_index) + {29, 31}, // (left_heel, left_foot_index) + {28, 32}, // (right_ankle, right_foot_index) + {27, 31}, // (left_ankle, left_foot_index) }}; } // namespace pose_landmarker diff --git a/mediapipe/tasks/testdata/vision/holistic_pose_tracking_graph.pbtxt b/mediapipe/tasks/testdata/vision/holistic_pose_tracking_graph.pbtxt new file mode 100644 index 000000000..7049f502f --- /dev/null +++ b/mediapipe/tasks/testdata/vision/holistic_pose_tracking_graph.pbtxt @@ -0,0 +1,238 @@ +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "LOOP:__stream_17" + input_stream: "MAIN:image_in" + output_stream: "PREV_LOOP:__stream_0" + input_stream_info { + tag_index: "LOOP" + back_edge: true + } +} +node { + calculator: "PacketPresenceCalculator" + input_stream: "PACKET:__stream_0" + output_stream: "PRESENCE:__stream_1" +} +node { + calculator: "GateCalculator" + input_stream: "image_in" + input_stream: "DISALLOW:__stream_1" + output_stream: "__stream_2" + options { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} +node { + calculator: "mediapipe.tasks.vision.pose_detector.PoseDetectorGraph" + input_stream: "IMAGE:__stream_2" + output_stream: "DETECTIONS:__stream_3" + options { + [mediapipe.tasks.vision.pose_detector.proto.PoseDetectorGraphOptions.ext] { + base_options { + model_asset { + file_name: "$0/mediapipe/tasks/testdata/vision/pose_detection.tflite" + } + } + num_poses: 1 + } + } +} +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:__stream_2" + output_stream: "SIZE:__stream_4" +} +node { + calculator: "AlignmentPointsRectsCalculator" + input_stream: "DETECTIONS:__stream_3" + input_stream: "IMAGE_SIZE:__stream_4" + output_stream: "NORM_RECT:__stream_5" + options { + [mediapipe.DetectionsToRectsCalculatorOptions.ext] { + rotation_vector_start_keypoint_index: 0 + rotation_vector_end_keypoint_index: 1 + rotation_vector_target_angle_degrees: 90 + } + } +} +node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_4" + input_stream: "NORM_RECT:__stream_5" + output_stream: "__stream_6" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 1.25 + scale_y: 1.25 + square_long: true + } + } +} +node { + calculator: "MergeCalculator" + input_stream: "__stream_6" + input_stream: "__stream_0" + output_stream: "__stream_7" +} +node { + calculator: "mediapipe.tasks.vision.pose_landmarker.SinglePoseLandmarksDetectorGraph" + input_stream: "IMAGE:image_in" + input_stream: "NORM_RECT:__stream_7" + output_stream: "AUXILIARY_LANDMARKS:__stream_8" + output_stream: "LANDMARKS:__stream_9" + output_stream: "WORLD_LANDMARKS:__stream_10" + options { + [mediapipe.tasks.vision.pose_landmarker.proto.PoseLandmarksDetectorGraphOptions.ext] { + base_options { + model_asset { + file_name: "$1/mediapipe/tasks/testdata/vision/pose_landmark_lite.tflite" + } + } + } + } +} +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image_in" + output_stream: "SIZE:__stream_11" +} +node { + calculator: "LandmarksToDetectionCalculator" + input_stream: "NORM_LANDMARKS:__stream_8" + output_stream: "DETECTION:__stream_12" +} +node { + calculator: "AlignmentPointsRectsCalculator" + input_stream: "DETECTION:__stream_12" + input_stream: "IMAGE_SIZE:__stream_11" + output_stream: "NORM_RECT:__stream_13" + options { + [mediapipe.DetectionsToRectsCalculatorOptions.ext] { + rotation_vector_start_keypoint_index: 0 + rotation_vector_end_keypoint_index: 1 + rotation_vector_target_angle_degrees: 90 + } + } +} +node { + calculator: "LandmarksSmoothingCalculator" + input_stream: "IMAGE_SIZE:__stream_11" + input_stream: "NORM_LANDMARKS:__stream_8" + input_stream: "OBJECT_SCALE_ROI:__stream_13" + output_stream: "NORM_FILTERED_LANDMARKS:__stream_14" + options { + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + one_euro_filter { + min_cutoff: 0.01 + beta: 10 + derivate_cutoff: 1 + disable_value_scaling: false + } + } + } +} +node { + calculator: "LandmarksToDetectionCalculator" + input_stream: "NORM_LANDMARKS:__stream_14" + output_stream: "DETECTION:__stream_15" +} +node { + calculator: "AlignmentPointsRectsCalculator" + input_stream: "DETECTION:__stream_15" + input_stream: "IMAGE_SIZE:__stream_11" + output_stream: "NORM_RECT:__stream_16" + options { + [mediapipe.DetectionsToRectsCalculatorOptions.ext] { + rotation_vector_start_keypoint_index: 0 + rotation_vector_end_keypoint_index: 1 + rotation_vector_target_angle_degrees: 90 + } + } +} +node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_11" + input_stream: "NORM_RECT:__stream_16" + output_stream: "__stream_17" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 1.25 + scale_y: 1.25 + square_long: true + } + } +} +node { + calculator: "VisibilitySmoothingCalculator" + input_stream: "NORM_LANDMARKS:__stream_9" + output_stream: "NORM_FILTERED_LANDMARKS:__stream_18" + options { + [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { + low_pass_filter { + alpha: 0.1 + } + } + } +} +node { + calculator: "LandmarksSmoothingCalculator" + input_stream: "IMAGE_SIZE:__stream_11" + input_stream: "NORM_LANDMARKS:__stream_18" + input_stream: "OBJECT_SCALE_ROI:__stream_13" + output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_out" + options { + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + one_euro_filter { + min_cutoff: 0.05 + beta: 80 + derivate_cutoff: 1 + disable_value_scaling: false + } + } + } +} +node { + calculator: "SplitLandmarkListCalculator" + input_stream: "__stream_10" + output_stream: "__stream_19" + options { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges { + begin: 0 + end: 33 + } + } + } +} +node { + calculator: "VisibilitySmoothingCalculator" + input_stream: "LANDMARKS:__stream_19" + output_stream: "FILTERED_LANDMARKS:__stream_20" + options { + [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { + low_pass_filter { + alpha: 0.1 + } + } + } +} +node { + calculator: "LandmarksSmoothingCalculator" + input_stream: "LANDMARKS:__stream_20" + output_stream: "FILTERED_LANDMARKS:pose_world_landmarks_out" + options { + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + one_euro_filter { + min_cutoff: 0.1 + beta: 40 + derivate_cutoff: 1 + disable_value_scaling: true + } + } + } +} +input_stream: "IMAGE:image_in" +output_stream: "POSE_LANDMARKS:pose_landmarks_out" +output_stream: "POSE_WORLD_LANDMARKS:pose_world_landmarks_out"