From d8006a2f87a3dc1bac49244683f68ad09ed841f6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 21 Oct 2022 21:52:40 -0700 Subject: [PATCH] Use model bundle for gesture recognizer. PiperOrigin-RevId: 482960305 --- .../tasks/cc/vision/gesture_recognizer/BUILD | 11 ++ .../gesture_recognizer/gesture_recognizer.cc | 46 ++---- .../gesture_recognizer/gesture_recognizer.h | 6 - .../gesture_recognizer_graph.cc | 71 +++++++++ .../hand_gesture_recognizer_graph.cc | 150 +++++++++++++++--- .../cc/vision/gesture_recognizer/proto/BUILD | 1 - ...and_gesture_recognizer_graph_options.proto | 7 +- .../hand_landmarker/hand_landmarker_graph.cc | 20 ++- .../hand_landmarker_graph_test.cc | 2 +- .../com/google/mediapipe/tasks/vision/BUILD | 1 + .../gesturerecognizer/GestureRecognizer.java | 53 ++----- .../GestureRecognizerTest.java | 141 +++++----------- mediapipe/tasks/testdata/vision/BUILD | 5 +- ...and_landmark.task => hand_landmarker.task} | Bin 7819037 -> 7819037 bytes third_party/external_files.bzl | 24 +-- 15 files changed, 309 insertions(+), 229 deletions(-) rename mediapipe/tasks/testdata/vision/{hand_landmark.task => hand_landmarker.task} (99%) diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD index a766c6b3f..6296017d4 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD @@ -62,13 +62,19 @@ cc_library( "//mediapipe/tasks/cc/components:image_preprocessing", "//mediapipe/tasks/cc/components/processors:classification_postprocessing_graph", "//mediapipe/tasks/cc/components/processors/proto:classification_postprocessing_graph_options_cc_proto", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", "//mediapipe/tasks/cc/core:model_resources", + "//mediapipe/tasks/cc/core:model_resources_cache", "//mediapipe/tasks/cc/core:model_task_graph", "//mediapipe/tasks/cc/core:utils", + "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", "//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto", + "//mediapipe/tasks/cc/metadata/utils:zip_utils", "//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:handedness_to_matrix_calculator", "//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:landmarks_to_matrix_calculator", "//mediapipe/tasks/cc/vision/gesture_recognizer/calculators:landmarks_to_matrix_calculator_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_embedder_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarks_detector_graph", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto", @@ -93,10 +99,14 @@ cc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:status", "//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", + "//mediapipe/tasks/cc/core:model_resources_cache", "//mediapipe/tasks/cc/core:model_task_graph", "//mediapipe/tasks/cc/core:utils", + "//mediapipe/tasks/cc/metadata/utils:zip_utils", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/hand_detector:hand_detector_graph", @@ -140,6 +150,7 @@ cc_library( "//mediapipe/tasks/cc/vision/core:image_processing_options", "//mediapipe/tasks/cc/vision/core:running_mode", "//mediapipe/tasks/cc/vision/core:vision_task_api_factory", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_cc_proto", diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc index 000a2e141..d4ab16ac8 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc @@ -41,6 +41,7 @@ limitations under the License. #include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h" #include "mediapipe/tasks/cc/vision/core/image_processing_options.h" #include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h" @@ -112,57 +113,38 @@ CalculatorGraphConfig CreateGraphConfig( std::unique_ptr ConvertGestureRecognizerGraphOptionsProto(GestureRecognizerOptions* options) { auto options_proto = std::make_unique(); + auto base_options_proto = std::make_unique( + tasks::core::ConvertBaseOptionsToProto(&(options->base_options))); + options_proto->mutable_base_options()->Swap(base_options_proto.get()); + options_proto->mutable_base_options()->set_use_stream_mode( + options->running_mode != core::RunningMode::IMAGE); - bool use_stream_mode = options->running_mode != core::RunningMode::IMAGE; - - // TODO remove these workarounds for base options of subgraphs. // Configure hand detector options. - auto base_options_proto_for_hand_detector = - std::make_unique( - tasks::core::ConvertBaseOptionsToProto( - &(options->base_options_for_hand_detector))); - base_options_proto_for_hand_detector->set_use_stream_mode(use_stream_mode); auto* hand_detector_graph_options = options_proto->mutable_hand_landmarker_graph_options() ->mutable_hand_detector_graph_options(); - hand_detector_graph_options->mutable_base_options()->Swap( - base_options_proto_for_hand_detector.get()); hand_detector_graph_options->set_num_hands(options->num_hands); hand_detector_graph_options->set_min_detection_confidence( options->min_hand_detection_confidence); // Configure hand landmark detector options. - auto base_options_proto_for_hand_landmarker = - std::make_unique( - tasks::core::ConvertBaseOptionsToProto( - &(options->base_options_for_hand_landmarker))); - base_options_proto_for_hand_landmarker->set_use_stream_mode(use_stream_mode); - auto* hand_landmarks_detector_graph_options = - options_proto->mutable_hand_landmarker_graph_options() - ->mutable_hand_landmarks_detector_graph_options(); - hand_landmarks_detector_graph_options->mutable_base_options()->Swap( - base_options_proto_for_hand_landmarker.get()); - hand_landmarks_detector_graph_options->set_min_detection_confidence( - options->min_hand_presence_confidence); - auto* hand_landmarker_graph_options = options_proto->mutable_hand_landmarker_graph_options(); hand_landmarker_graph_options->set_min_tracking_confidence( options->min_tracking_confidence); + auto* hand_landmarks_detector_graph_options = + hand_landmarker_graph_options + ->mutable_hand_landmarks_detector_graph_options(); + hand_landmarks_detector_graph_options->set_min_detection_confidence( + options->min_hand_presence_confidence); // Configure hand gesture recognizer options. - auto base_options_proto_for_gesture_recognizer = - std::make_unique( - tasks::core::ConvertBaseOptionsToProto( - &(options->base_options_for_gesture_recognizer))); - base_options_proto_for_gesture_recognizer->set_use_stream_mode( - use_stream_mode); auto* hand_gesture_recognizer_graph_options = options_proto->mutable_hand_gesture_recognizer_graph_options(); - hand_gesture_recognizer_graph_options->mutable_base_options()->Swap( - base_options_proto_for_gesture_recognizer.get()); if (options->min_gesture_confidence >= 0) { - hand_gesture_recognizer_graph_options->mutable_classifier_options() + hand_gesture_recognizer_graph_options + ->mutable_canned_gesture_classifier_graph_options() + ->mutable_classifier_options() ->set_score_threshold(options->min_gesture_confidence); } return options_proto; diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h index 29c8bea7b..3e281b26e 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h @@ -39,12 +39,6 @@ struct GestureRecognizerOptions { // model file with metadata, accelerator options, op resolver, etc. tasks::core::BaseOptions base_options; - // TODO: remove these. Temporary solutions before bundle asset is - // ready. - tasks::core::BaseOptions base_options_for_hand_landmarker; - tasks::core::BaseOptions base_options_for_hand_detector; - tasks::core::BaseOptions base_options_for_gesture_recognizer; - // The running mode of the task. Default to the image mode. // GestureRecognizer has three running modes: // 1) The image mode for recognizing hand gestures on single image inputs. diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc index e02eadde8..7ab4847dd 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc @@ -25,9 +25,13 @@ limitations under the License. #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/tasks/cc/common.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" +#include "mediapipe/tasks/cc/core/model_resources_cache.h" #include "mediapipe/tasks/cc/core/model_task_graph.h" #include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h" @@ -46,6 +50,8 @@ using ::mediapipe::api2::Input; using ::mediapipe::api2::Output; using ::mediapipe::api2::builder::Graph; using ::mediapipe::api2::builder::Source; +using ::mediapipe::tasks::core::ModelAssetBundleResources; +using ::mediapipe::tasks::metadata::SetExternalFile; using ::mediapipe::tasks::vision::gesture_recognizer::proto:: GestureRecognizerGraphOptions; using ::mediapipe::tasks::vision::gesture_recognizer::proto:: @@ -61,6 +67,9 @@ constexpr char kHandednessTag[] = "HANDEDNESS"; constexpr char kImageSizeTag[] = "IMAGE_SIZE"; constexpr char kHandGesturesTag[] = "HAND_GESTURES"; constexpr char kHandTrackingIdsTag[] = "HAND_TRACKING_IDS"; +constexpr char kHandLandmarkerBundleAssetName[] = "hand_landmarker.task"; +constexpr char kHandGestureRecognizerBundleAssetName[] = + "hand_gesture_recognizer.task"; struct GestureRecognizerOutputs { Source> gesture; @@ -70,6 +79,53 @@ struct GestureRecognizerOutputs { Source image; }; +// Sets the base options in the sub tasks. +absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, + GestureRecognizerGraphOptions* options, + bool is_copy) { + ASSIGN_OR_RETURN(const auto hand_landmarker_file, + resources.GetModelFile(kHandLandmarkerBundleAssetName)); + auto* hand_landmarker_graph_options = + options->mutable_hand_landmarker_graph_options(); + SetExternalFile(hand_landmarker_file, + hand_landmarker_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + hand_landmarker_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + hand_landmarker_graph_options->mutable_base_options()->set_use_stream_mode( + options->base_options().use_stream_mode()); + + ASSIGN_OR_RETURN( + const auto hand_gesture_recognizer_file, + resources.GetModelFile(kHandGestureRecognizerBundleAssetName)); + auto* hand_gesture_recognizer_graph_options = + options->mutable_hand_gesture_recognizer_graph_options(); + SetExternalFile(hand_gesture_recognizer_file, + hand_gesture_recognizer_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + hand_gesture_recognizer_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + if (!hand_gesture_recognizer_graph_options->base_options() + .acceleration() + .has_xnnpack() && + !hand_gesture_recognizer_graph_options->base_options() + .acceleration() + .has_tflite()) { + hand_gesture_recognizer_graph_options->mutable_base_options() + ->mutable_acceleration() + ->mutable_xnnpack(); + LOG(WARNING) << "Hand Gesture Recognizer contains CPU only ops. Sets " + << "HandGestureRecognizerGraph acceleartion to Xnnpack."; + } + hand_gesture_recognizer_graph_options->mutable_base_options() + ->set_use_stream_mode(options->base_options().use_stream_mode()); + return absl::OkStatus(); +} + } // namespace // A "mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph" performs @@ -136,6 +192,21 @@ class GestureRecognizerGraph : public core::ModelTaskGraph { absl::StatusOr GetConfig( SubgraphContext* sc) override { Graph graph; + if (sc->Options() + .base_options() + .has_model_asset()) { + ASSIGN_OR_RETURN( + const auto* model_asset_bundle_resources, + CreateModelAssetBundleResources(sc)); + // When the model resources cache service is available, filling in + // the file pointer meta in the subtasks' base options. Otherwise, + // providing the file contents instead. + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + *model_asset_bundle_resources, + sc->MutableOptions(), + !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) + .IsAvailable())); + } ASSIGN_OR_RETURN(auto hand_gesture_recognition_output, BuildGestureRecognizerGraph( *sc->MutableOptions(), diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc index 4bbe94974..7b7746956 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc @@ -30,11 +30,17 @@ limitations under the License. #include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.h" #include "mediapipe/tasks/cc/components/processors/proto/classification_postprocessing_graph_options.pb.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" #include "mediapipe/tasks/cc/core/model_resources.h" +#include "mediapipe/tasks/cc/core/model_resources_cache.h" #include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" #include "mediapipe/tasks/cc/core/proto/inference_subgraph.pb.h" #include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/calculators/landmarks_to_matrix_calculator.pb.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_embedder_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h" #include "mediapipe/tasks/metadata/metadata_schema_generated.h" @@ -51,6 +57,8 @@ using ::mediapipe::api2::builder::Graph; using ::mediapipe::api2::builder::Source; using ::mediapipe::tasks::components::processors:: ConfigureTensorsToClassificationCalculator; +using ::mediapipe::tasks::core::ModelAssetBundleResources; +using ::mediapipe::tasks::metadata::SetExternalFile; using ::mediapipe::tasks::vision::gesture_recognizer::proto:: HandGestureRecognizerGraphOptions; @@ -70,6 +78,14 @@ constexpr char kVectorTag[] = "VECTOR"; constexpr char kIndexTag[] = "INDEX"; constexpr char kIterableTag[] = "ITERABLE"; constexpr char kBatchEndTag[] = "BATCH_END"; +constexpr char kGestureEmbedderTFLiteName[] = "gesture_embedder.tflite"; +constexpr char kCannedGestureClassifierTFLiteName[] = + "canned_gesture_classifier.tflite"; + +struct SubTaskModelResources { + const core::ModelResources* gesture_embedder_model_resource; + const core::ModelResources* canned_gesture_classifier_model_resource; +}; Source> ConvertMatrixToTensor(Source matrix, Graph& graph) { @@ -78,6 +94,41 @@ Source> ConvertMatrixToTensor(Source matrix, return node[Output>{"TENSORS"}]; } +// Sets the base options in the sub tasks. +absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, + HandGestureRecognizerGraphOptions* options, + bool is_copy) { + ASSIGN_OR_RETURN(const auto gesture_embedder_file, + resources.GetModelFile(kGestureEmbedderTFLiteName)); + auto* gesture_embedder_graph_options = + options->mutable_gesture_embedder_graph_options(); + SetExternalFile(gesture_embedder_file, + gesture_embedder_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + gesture_embedder_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + gesture_embedder_graph_options->mutable_base_options()->set_use_stream_mode( + options->base_options().use_stream_mode()); + + ASSIGN_OR_RETURN(const auto canned_gesture_classifier_file, + resources.GetModelFile(kCannedGestureClassifierTFLiteName)); + auto* canned_gesture_classifier_graph_options = + options->mutable_canned_gesture_classifier_graph_options(); + SetExternalFile( + canned_gesture_classifier_file, + canned_gesture_classifier_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + canned_gesture_classifier_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + canned_gesture_classifier_graph_options->mutable_base_options() + ->set_use_stream_mode(options->base_options().use_stream_mode()); + return absl::OkStatus(); +} + } // namespace // A @@ -128,27 +179,70 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph { public: absl::StatusOr GetConfig( SubgraphContext* sc) override { - ASSIGN_OR_RETURN( - const auto* model_resources, - CreateModelResources(sc)); + if (sc->Options() + .base_options() + .has_model_asset()) { + ASSIGN_OR_RETURN( + const auto* model_asset_bundle_resources, + CreateModelAssetBundleResources( + sc)); + // When the model resources cache service is available, filling in + // the file pointer meta in the subtasks' base options. Otherwise, + // providing the file contents instead. + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + *model_asset_bundle_resources, + sc->MutableOptions(), + !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) + .IsAvailable())); + } + ASSIGN_OR_RETURN(const auto sub_task_model_resources, + CreateSubTaskModelResources(sc)); Graph graph; - ASSIGN_OR_RETURN( - auto hand_gestures, - BuildGestureRecognizerGraph( - sc->Options(), *model_resources, - graph[Input(kHandednessTag)], - graph[Input(kLandmarksTag)], - graph[Input(kWorldLandmarksTag)], - graph[Input>(kImageSizeTag)], - graph[Input(kNormRectTag)], graph)); + ASSIGN_OR_RETURN(auto hand_gestures, + BuildGestureRecognizerGraph( + sc->Options(), + sub_task_model_resources, + graph[Input(kHandednessTag)], + graph[Input(kLandmarksTag)], + graph[Input(kWorldLandmarksTag)], + graph[Input>(kImageSizeTag)], + graph[Input(kNormRectTag)], graph)); hand_gestures >> graph[Output(kHandGesturesTag)]; return graph.GetConfig(); } private: + absl::StatusOr CreateSubTaskModelResources( + SubgraphContext* sc) { + auto* options = sc->MutableOptions(); + SubTaskModelResources sub_task_model_resources; + auto& gesture_embedder_model_asset = + *options->mutable_gesture_embedder_graph_options() + ->mutable_base_options() + ->mutable_model_asset(); + ASSIGN_OR_RETURN( + sub_task_model_resources.gesture_embedder_model_resource, + CreateModelResources(sc, + std::make_unique( + std::move(gesture_embedder_model_asset)), + "_gesture_embedder")); + auto& canned_gesture_classifier_model_asset = + *options->mutable_canned_gesture_classifier_graph_options() + ->mutable_base_options() + ->mutable_model_asset(); + ASSIGN_OR_RETURN( + sub_task_model_resources.canned_gesture_classifier_model_resource, + CreateModelResources( + sc, + std::make_unique( + std::move(canned_gesture_classifier_model_asset)), + "_canned_gesture_classifier")); + return sub_task_model_resources; + } + absl::StatusOr> BuildGestureRecognizerGraph( const HandGestureRecognizerGraphOptions& graph_options, - const core::ModelResources& model_resources, + const SubTaskModelResources& sub_task_model_resources, Source handedness, Source hand_landmarks, Source hand_world_landmarks, @@ -209,17 +303,33 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph { auto concatenated_tensors = concatenate_tensor_vector.Out(""); // Inference for static hand gesture recognition. - // TODO add embedding step. - auto& inference = AddInference( - model_resources, graph_options.base_options().acceleration(), graph); - concatenated_tensors >> inference.In(kTensorsTag); - auto inference_output_tensors = inference.Out(kTensorsTag); + auto& gesture_embedder_inference = + AddInference(*sub_task_model_resources.gesture_embedder_model_resource, + graph_options.gesture_embedder_graph_options() + .base_options() + .acceleration(), + graph); + concatenated_tensors >> gesture_embedder_inference.In(kTensorsTag); + auto embedding_tensors = gesture_embedder_inference.Out(kTensorsTag); + + auto& canned_gesture_classifier_inference = AddInference( + *sub_task_model_resources.canned_gesture_classifier_model_resource, + graph_options.canned_gesture_classifier_graph_options() + .base_options() + .acceleration(), + graph); + embedding_tensors >> canned_gesture_classifier_inference.In(kTensorsTag); + auto inference_output_tensors = + canned_gesture_classifier_inference.Out(kTensorsTag); auto& tensors_to_classification = graph.AddNode("TensorsToClassificationCalculator"); MP_RETURN_IF_ERROR(ConfigureTensorsToClassificationCalculator( - graph_options.classifier_options(), - *model_resources.GetMetadataExtractor(), 0, + graph_options.canned_gesture_classifier_graph_options() + .classifier_options(), + *sub_task_model_resources.canned_gesture_classifier_model_resource + ->GetMetadataExtractor(), + 0, &tensors_to_classification.GetOptions< mediapipe::TensorsToClassificationCalculatorOptions>())); inference_output_tensors >> tensors_to_classification.In(kTensorsTag); diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/BUILD b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/BUILD index 3b73bf2b0..0db47da7a 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/BUILD +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/BUILD @@ -49,7 +49,6 @@ mediapipe_proto_library( ":gesture_embedder_graph_options_proto", "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", - "//mediapipe/tasks/cc/components/processors/proto:classifier_options_proto", "//mediapipe/tasks/cc/core/proto:base_options_proto", ], ) diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto index a3281702a..7df2fed37 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto @@ -18,7 +18,6 @@ syntax = "proto2"; package mediapipe.tasks.vision.gesture_recognizer.proto; import "mediapipe/framework/calculator.proto"; -import "mediapipe/tasks/cc/components/processors/proto/classifier_options.proto"; import "mediapipe/tasks/cc/core/proto/base_options.proto"; import "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.proto"; import "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_embedder_graph_options.proto"; @@ -37,15 +36,11 @@ message HandGestureRecognizerGraphOptions { // Options for GestureEmbedder. optional GestureEmbedderGraphOptions gesture_embedder_graph_options = 2; - // Options for GestureClassifier of default gestures. + // Options for GestureClassifier of canned gestures. optional GestureClassifierGraphOptions canned_gesture_classifier_graph_options = 3; // Options for GestureClassifier of custom gestures. optional GestureClassifierGraphOptions custom_gesture_classifier_graph_options = 4; - - // TODO: remove these. Temporary solutions before bundle asset is - // ready. - optional components.processors.proto.ClassifierOptions classifier_options = 5; } diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc index 3fbe38c1c..e610a412e 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc @@ -92,18 +92,30 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, bool is_copy) { ASSIGN_OR_RETURN(const auto hand_detector_file, resources.GetModelFile(kHandDetectorTFLiteName)); + auto* hand_detector_graph_options = + options->mutable_hand_detector_graph_options(); SetExternalFile(hand_detector_file, - options->mutable_hand_detector_graph_options() - ->mutable_base_options() + hand_detector_graph_options->mutable_base_options() ->mutable_model_asset(), is_copy); + hand_detector_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + hand_detector_graph_options->mutable_base_options()->set_use_stream_mode( + options->base_options().use_stream_mode()); ASSIGN_OR_RETURN(const auto hand_landmarks_detector_file, resources.GetModelFile(kHandLandmarksDetectorTFLiteName)); + auto* hand_landmarks_detector_graph_options = + options->mutable_hand_landmarks_detector_graph_options(); SetExternalFile(hand_landmarks_detector_file, - options->mutable_hand_landmarks_detector_graph_options() - ->mutable_base_options() + hand_landmarks_detector_graph_options->mutable_base_options() ->mutable_model_asset(), is_copy); + hand_landmarks_detector_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + hand_landmarks_detector_graph_options->mutable_base_options() + ->set_use_stream_mode(options->base_options().use_stream_mode()); return absl::OkStatus(); } diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc index 08beb1a1b..f275486f5 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc @@ -67,7 +67,7 @@ using ::testing::proto::Approximately; using ::testing::proto::Partially; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; -constexpr char kHandLandmarkerModelBundle[] = "hand_landmark.task"; +constexpr char kHandLandmarkerModelBundle[] = "hand_landmarker.task"; constexpr char kLeftHandsImage[] = "left_hands.jpg"; constexpr char kLeftHandsRotatedImage[] = "left_hands_rotated.jpg"; diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index dcf3b3542..2bdcc2522 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -128,6 +128,7 @@ android_library( "//mediapipe/java/com/google/mediapipe/framework/image", "//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite", "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java index 560508903..55cf275e9 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java @@ -38,6 +38,7 @@ import com.google.mediapipe.tasks.core.TaskRunner; import com.google.mediapipe.tasks.core.proto.BaseOptionsProto; import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi; import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.GestureClassifierGraphOptionsProto; import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.GestureRecognizerGraphOptionsProto; import com.google.mediapipe.tasks.vision.gesturerecognizer.proto.HandGestureRecognizerGraphOptionsProto; import com.google.mediapipe.tasks.vision.handdetector.proto.HandDetectorGraphOptionsProto; @@ -300,13 +301,6 @@ public final class GestureRecognizer extends BaseVisionTaskApi { */ public abstract Builder setRunningMode(RunningMode value); - // TODO: remove these. Temporary solutions before bundle asset is ready. - public abstract Builder setBaseOptionsHandDetector(BaseOptions value); - - public abstract Builder setBaseOptionsHandLandmarker(BaseOptions value); - - public abstract Builder setBaseOptionsGestureRecognizer(BaseOptions value); - /** Sets the maximum number of hands can be detected by the GestureRecognizer. */ public abstract Builder setNumHands(Integer value); @@ -366,13 +360,6 @@ public final class GestureRecognizer extends BaseVisionTaskApi { abstract BaseOptions baseOptions(); - // TODO: remove these. Temporary solutions before bundle asset is ready. - abstract BaseOptions baseOptionsHandDetector(); - - abstract BaseOptions baseOptionsHandLandmarker(); - - abstract BaseOptions baseOptionsGestureRecognizer(); - abstract RunningMode runningMode(); abstract Optional numHands(); @@ -405,22 +392,18 @@ public final class GestureRecognizer extends BaseVisionTaskApi { */ @Override public CalculatorOptions convertToCalculatorOptionsProto() { - BaseOptionsProto.BaseOptions.Builder baseOptionsBuilder = - BaseOptionsProto.BaseOptions.newBuilder() - .setUseStreamMode(runningMode() != RunningMode.IMAGE) - .mergeFrom(convertBaseOptionsToProto(baseOptions())); GestureRecognizerGraphOptionsProto.GestureRecognizerGraphOptions.Builder taskOptionsBuilder = GestureRecognizerGraphOptionsProto.GestureRecognizerGraphOptions.newBuilder() - .setBaseOptions(baseOptionsBuilder); + .setBaseOptions( + BaseOptionsProto.BaseOptions.newBuilder() + .setUseStreamMode(runningMode() != RunningMode.IMAGE) + .mergeFrom(convertBaseOptionsToProto(baseOptions())) + .build()); // Setup HandDetectorGraphOptions. HandDetectorGraphOptionsProto.HandDetectorGraphOptions.Builder handDetectorGraphOptionsBuilder = - HandDetectorGraphOptionsProto.HandDetectorGraphOptions.newBuilder() - .setBaseOptions( - BaseOptionsProto.BaseOptions.newBuilder() - .setUseStreamMode(runningMode() != RunningMode.IMAGE) - .mergeFrom(convertBaseOptionsToProto(baseOptionsHandDetector()))); + HandDetectorGraphOptionsProto.HandDetectorGraphOptions.newBuilder(); numHands().ifPresent(handDetectorGraphOptionsBuilder::setNumHands); minHandDetectionConfidence() .ifPresent(handDetectorGraphOptionsBuilder::setMinDetectionConfidence); @@ -428,19 +411,12 @@ public final class GestureRecognizer extends BaseVisionTaskApi { // Setup HandLandmarkerGraphOptions. HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.Builder handLandmarksDetectorGraphOptionsBuilder = - HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.newBuilder() - .setBaseOptions( - BaseOptionsProto.BaseOptions.newBuilder() - .setUseStreamMode(runningMode() != RunningMode.IMAGE) - .mergeFrom(convertBaseOptionsToProto(baseOptionsHandLandmarker()))); + HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions.newBuilder(); minHandPresenceConfidence() .ifPresent(handLandmarksDetectorGraphOptionsBuilder::setMinDetectionConfidence); HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.Builder handLandmarkerGraphOptionsBuilder = - HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.newBuilder() - .setBaseOptions( - BaseOptionsProto.BaseOptions.newBuilder() - .setUseStreamMode(runningMode() != RunningMode.IMAGE)); + HandLandmarkerGraphOptionsProto.HandLandmarkerGraphOptions.newBuilder(); minTrackingConfidence() .ifPresent(handLandmarkerGraphOptionsBuilder::setMinTrackingConfidence); handLandmarkerGraphOptionsBuilder @@ -450,16 +426,13 @@ public final class GestureRecognizer extends BaseVisionTaskApi { // Setup HandGestureRecognizerGraphOptions. HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.Builder handGestureRecognizerGraphOptionsBuilder = - HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder() - .setBaseOptions( - BaseOptionsProto.BaseOptions.newBuilder() - .setUseStreamMode(runningMode() != RunningMode.IMAGE) - .mergeFrom(convertBaseOptionsToProto(baseOptionsGestureRecognizer()))); + HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder(); ClassifierOptionsProto.ClassifierOptions.Builder classifierOptionsBuilder = ClassifierOptionsProto.ClassifierOptions.newBuilder(); minGestureConfidence().ifPresent(classifierOptionsBuilder::setScoreThreshold); - handGestureRecognizerGraphOptionsBuilder.setClassifierOptions( - classifierOptionsBuilder.build()); + handGestureRecognizerGraphOptionsBuilder.setCannedGestureClassifierGraphOptions( + GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder() + .setClassifierOptions(classifierOptionsBuilder.build())); taskOptionsBuilder .setHandLandmarkerGraphOptions(handLandmarkerGraphOptionsBuilder.build()) diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java index 8beea96ac..31e59a259 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java @@ -43,10 +43,7 @@ import org.junit.runners.Suite.SuiteClasses; @RunWith(Suite.class) @SuiteClasses({GestureRecognizerTest.General.class, GestureRecognizerTest.RunningModeTest.class}) public class GestureRecognizerTest { - private static final String HAND_DETECTOR_MODEL_FILE = "palm_detection_full.tflite"; - private static final String HAND_LANDMARKER_MODEL_FILE = "hand_landmark_full.tflite"; - private static final String GESTURE_RECOGNIZER_MODEL_FILE = - "cg_classifier_screen3d_landmark_features_nn_2022_08_04_base_simple_model.tflite"; + private static final String GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE = "gesture_recognizer.task"; private static final String TWO_HANDS_IMAGE = "right_hands.jpg"; private static final String THUMB_UP_IMAGE = "thumb_up.jpg"; private static final String NO_HANDS_IMAGE = "cats_and_dogs.jpg"; @@ -66,13 +63,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .build(); GestureRecognizer gestureRecognizer = GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); @@ -88,13 +81,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .build(); GestureRecognizer gestureRecognizer = GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); @@ -111,16 +100,12 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) // TODO update the confidence to be in range [0,1] after embedding model // and scoring calculator is integrated. - .setMinGestureConfidence(3.0f) + .setMinGestureConfidence(2.0f) .build(); GestureRecognizer gestureRecognizer = GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options); @@ -139,13 +124,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setNumHands(2) .build(); GestureRecognizer gestureRecognizer = @@ -168,19 +149,7 @@ public class GestureRecognizerTest { GestureRecognizerOptions.builder() .setBaseOptions( BaseOptions.builder() - .setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE) - .build()) - .setBaseOptionsHandDetector( - BaseOptions.builder() - .setModelAssetPath(HAND_DETECTOR_MODEL_FILE) - .build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder() - .setModelAssetPath(HAND_LANDMARKER_MODEL_FILE) - .build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder() - .setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE) + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) .build()) .setRunningMode(mode) .setResultListener((gestureRecognitionResult, inputImage) -> {}) @@ -201,15 +170,7 @@ public class GestureRecognizerTest { GestureRecognizerOptions.builder() .setBaseOptions( BaseOptions.builder() - .setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE) - .build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder() - .setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE) + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) .build()) .setRunningMode(RunningMode.LIVE_STREAM) .build()); @@ -223,13 +184,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.IMAGE) .build(); @@ -252,13 +209,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.VIDEO) .build(); @@ -281,13 +234,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.LIVE_STREAM) .setResultListener((gestureRecognitionResult, inputImage) -> {}) .build(); @@ -311,13 +260,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.IMAGE) .build(); @@ -335,13 +280,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.VIDEO) .build(); GestureRecognizer gestureRecognizer = @@ -363,13 +304,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.LIVE_STREAM) .setResultListener( (actualResult, inputImage) -> { @@ -397,13 +334,9 @@ public class GestureRecognizerTest { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) - .setBaseOptionsHandDetector( - BaseOptions.builder().setModelAssetPath(HAND_DETECTOR_MODEL_FILE).build()) - .setBaseOptionsHandLandmarker( - BaseOptions.builder().setModelAssetPath(HAND_LANDMARKER_MODEL_FILE).build()) - .setBaseOptionsGestureRecognizer( - BaseOptions.builder().setModelAssetPath(GESTURE_RECOGNIZER_MODEL_FILE).build()) + BaseOptions.builder() + .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE) + .build()) .setRunningMode(RunningMode.LIVE_STREAM) .setResultListener( (actualResult, inputImage) -> { diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index f899be8ef..ebb8f05a6 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -35,7 +35,6 @@ mediapipe_files(srcs = [ "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite", "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29_with_dummy_score_calibration.tflite", "deeplabv3.tflite", - "hand_landmark.task", "hand_landmark_full.tflite", "hand_landmark_lite.tflite", "left_hands.jpg", @@ -67,13 +66,13 @@ mediapipe_files(srcs = [ exports_files( srcs = [ - "cg_classifier_screen3d_landmark_features_nn_2022_08_04_base_simple_model.tflite", "expected_left_down_hand_landmarks.prototxt", "expected_left_down_hand_rotated_landmarks.prototxt", "expected_left_up_hand_landmarks.prototxt", "expected_left_up_hand_rotated_landmarks.prototxt", "expected_right_down_hand_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt", + "gesture_recognizer.task", ], ) @@ -119,9 +118,9 @@ filegroup( "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite", "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29_with_dummy_score_calibration.tflite", "deeplabv3.tflite", - "hand_landmark.task", "hand_landmark_full.tflite", "hand_landmark_lite.tflite", + "hand_landmarker.task", "mobilenet_v1_0.25_192_quantized_1_default_1.tflite", "mobilenet_v1_0.25_224_1_default_1.tflite", "mobilenet_v1_0.25_224_1_metadata_1.tflite", diff --git a/mediapipe/tasks/testdata/vision/hand_landmark.task b/mediapipe/tasks/testdata/vision/hand_landmarker.task similarity index 99% rename from mediapipe/tasks/testdata/vision/hand_landmark.task rename to mediapipe/tasks/testdata/vision/hand_landmarker.task index b6eedf3248f860d736929631c3be3d6b0f9d4cdc..1ae9f7f6b33c52e16493b566cc76d014655c4d25 100644 GIT binary patch delta 491 zcmW;I*)ALa7{zgns?t-%sHL#3D#Wm(x)(8{r%S-Do8m2VYT&#gkM$a-PDw2G|~tMpo-rdWW3f5Nd4%2199 zRH6#6@EULM7S(u%8q}f=^=LpNn$V0Ew4x2|c#jTr;sd(SjgRO-FZvKhKL#*}Pxy=> z3}XZljA9Jqi1&};3nnpzY0Tg&zF`(~n8yMZv4mx;U=?dv#|AdBg>CHMJAPmnd)UVT e4snE^IL5Dk;n?r)^Y~ZK{zgu^?!!&wB>ErZ3Beiw delta 491 zcmW;I>oODo7{+l~FPJ|qGrA5h7az5k~5;@z?K0L!6m}zFaiZ$JXd+1*L z*E{o@c|N=|?=x@w@Bcs?nj8zH6{RF6w{iSJ)&ncm3Rr1Yx|Lx)v@)$ME8EJkaxK>K ztbFT{_1G$~3auim*ebC~tupJ0Rc=*SPpwL;%Br@WS~E_G@=R3XhAF5@EYxSgAR1!ExOQ+9=t;@`p}O7gb>Cc-eU+K5Wz4;Fp7`(gfV