diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD index 8097d7ab1..6f79b9c4f 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD @@ -35,6 +35,7 @@ mediapipe_proto_library( srcs = ["hand_landmarker_graph_options.proto"], deps = [ ":hand_landmarks_detector_graph_options_proto", + ":hand_roi_refinement_graph_options_proto", "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/tasks/cc/core/proto:base_options_proto", @@ -42,4 +43,12 @@ mediapipe_proto_library( ], ) +mediapipe_proto_library( + name = "hand_roi_refinement_graph_options_proto", + srcs = ["hand_roi_refinement_graph_options.proto"], + deps = [ + "//mediapipe/tasks/cc/core/proto:base_options_proto", + ], +) + # TODO: open source hand joints graph diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.proto b/mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.proto new file mode 100644 index 000000000..0f979c5aa --- /dev/null +++ b/mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.proto @@ -0,0 +1,24 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.vision.hand_landmarker.proto; + +import "mediapipe/tasks/cc/core/proto/base_options.proto"; + +message HandRoiRefinementGraphOptions { + core.proto.BaseOptions base_options = 1; +} diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 19c9e6132..89c59ec3b 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -55,6 +55,7 @@ mediapipe_files(srcs = [ "hand_landmark_full.tflite", "hand_landmark_lite.tflite", "hand_landmarker.task", + "handrecrop_2020_07_21_v0.f16.tflite", "left_hands.jpg", "left_hands_rotated.jpg", "leopard_bg_removal_result_512x512.png", @@ -190,6 +191,7 @@ filegroup( "hand_landmark_full.tflite", "hand_landmark_lite.tflite", "hand_landmarker.task", + "handrecrop_2020_07_21_v0.f16.tflite", "mobilenet_v1_0.25_192_quantized_1_default_1.tflite", "mobilenet_v1_0.25_224_1_default_1.tflite", "mobilenet_v1_0.25_224_1_metadata_1.tflite", diff --git a/mediapipe/tasks/testdata/vision/hand_roi_refinement_generated_graph.pbtxt b/mediapipe/tasks/testdata/vision/hand_roi_refinement_generated_graph.pbtxt new file mode 100644 index 000000000..06d9f07d1 --- /dev/null +++ b/mediapipe/tasks/testdata/vision/hand_roi_refinement_generated_graph.pbtxt @@ -0,0 +1,131 @@ +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph__ImageCloneCalculator" + calculator: "ImageCloneCalculator" + input_stream: "image" + output_stream: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph____stream_5" + options { + [mediapipe.ImageCloneCalculatorOptions.ext] { + output_on_gpu: false + } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph__ImageToTensorCalculator" + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph____stream_5" + input_stream: "NORM_RECT:roi" + output_stream: "LETTERBOX_PADDING:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph____stream_2" + output_stream: "MATRIX:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_3" + output_stream: "TENSORS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_4" + options { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 256 + output_tensor_height: 256 + keep_aspect_ratio: true + output_tensor_float_range { + min: 0 + max: 1 + } + gpu_origin: TOP_LEFT + border_mode: BORDER_REPLICATE + } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph__ImagePropertiesCalculator" + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_2" +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph__PassThroughCalculator" + calculator: "PassThroughCalculator" + input_stream: "image" + output_stream: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_components_processors_imagepreprocessinggraph____stream_7" +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph__ModelResourcesCalculator" + calculator: "ModelResourcesCalculator" + output_side_packet: "METADATA_EXTRACTOR:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_1" + output_side_packet: "MODEL:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2" + output_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3" + options { + [mediapipe.tasks.core.proto.ModelResourcesCalculatorOptions.ext] { + model_file { + file_name: "mediapipe/tasks/testdata/vision/handrecrop_2020_07_21_v0.f16.tflite" + } + } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph__inferencecalculator__mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph__InferenceCalculator" + calculator: "InferenceCalculatorCpu" + input_stream: "TENSORS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_4" + output_stream: "TENSORS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_5" + input_side_packet: "MODEL:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2" + input_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3" + options { + [mediapipe.InferenceCalculatorOptions.ext] { delegate {} } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__TensorsToLandmarksCalculator" + calculator: "TensorsToLandmarksCalculator" + input_stream: "TENSORS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_5" + output_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_6" + options { + [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { + num_landmarks: 2 + input_image_width: 256 + input_image_height: 256 + normalize_z: 1 + } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__LandmarkProjectionCalculator" + calculator: "LandmarkProjectionCalculator" + input_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_6" + input_stream: "PROJECTION_MATRIX:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_3" + output_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_7" +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__LandmarksToDetectionCalculator" + calculator: "LandmarksToDetectionCalculator" + input_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_7" + output_stream: "DETECTION:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_8" +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__AlignmentPointsRectsCalculator" + calculator: "AlignmentPointsRectsCalculator" + input_stream: "DETECTION:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_8" + input_stream: "IMAGE_SIZE:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_2" + output_stream: "NORM_RECT:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_9" + options { + [mediapipe.DetectionsToRectsCalculatorOptions.ext] { + rotation_vector_start_keypoint_index: 0 + rotation_vector_end_keypoint_index: 1 + rotation_vector_target_angle_degrees: -90 + } + } +} +node { + name: "mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph__RectTransformationCalculator" + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_2" + input_stream: "NORM_RECT:mediapipe_tasks_vision_hand_landmarker_handroirefinementgraph____stream_9" + output_stream: "refined_roi" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 1 + scale_y: 1 + shift_x: 0 + shift_y: -0.1 + square_long: true + } + } +} +input_stream: "IMAGE:image" +input_stream: "NORM_RECT:roi" +executor { +} diff --git a/third_party/external_files.bzl b/third_party/external_files.bzl index d3ec11f95..750421b6e 100644 --- a/third_party/external_files.bzl +++ b/third_party/external_files.bzl @@ -532,12 +532,24 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/hand_landmark.tflite?generation=1666153735814956"], ) + http_file( + name = "com_google_mediapipe_handrecrop_2020_07_21_v0_f16_tflite", + sha256 = "d40b15e15f93f6c909a3cfb881ce16c9ff9aa6d57417a0c906a6624f1f60b60c", + urls = ["https://storage.googleapis.com/mediapipe-assets/handrecrop_2020_07_21_v0.f16.tflite?generation=1695159193831748"], + ) + http_file( name = "com_google_mediapipe_hand_recrop_tflite", sha256 = "67d996ce96f9d36fe17d2693022c6da93168026ab2f028f9e2365398d8ac7d5d", urls = ["https://storage.googleapis.com/mediapipe-assets/hand_recrop.tflite?generation=1661875770633070"], ) + http_file( + name = "com_google_mediapipe_hand_roi_refinement_generated_graph_pbtxt", + sha256 = "a2304dedc6f1d167996d7261cd5b5e5db843c3b5657a367d9341388f350bdcc2", + urls = ["https://storage.googleapis.com/mediapipe-assets/hand_roi_refinement_generated_graph.pbtxt?generation=1695159196033618"], + ) + http_file( name = "com_google_mediapipe_image_tensor_meta_json", sha256 = "aad86fde3defb379c82ff7ee48e50493a58529cdc0623cf0d7bf135c3577060e",