Merge branch 'google:master' into gesture-recognizer-python

mediapipe/calculators/core/BUILD

@@ -222,10 +222,10 @@ cc_library(
         "//mediapipe/framework:calculator_contract",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:collection_item_id",
-        "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",

mediapipe/calculators/core/end_loop_calculator.cc

@@ -19,6 +19,7 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
 #include "tensorflow/lite/interpreter.h"
@@ -58,4 +59,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
     EndLoopDetectionCalculator;
 REGISTER_CALCULATOR(EndLoopDetectionCalculator);
 
+typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator;
+REGISTER_CALCULATOR(EndLoopMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/tasks/cc/vision/gesture_recognizer/BUILD

@@ -141,6 +141,7 @@ cc_library(
         "//mediapipe/tasks/cc:common",
         "//mediapipe/tasks/cc/components:image_preprocessing",
         "//mediapipe/tasks/cc/components/containers:gesture_recognition_result",
+        "//mediapipe/tasks/cc/components/processors:classifier_options",
         "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
         "//mediapipe/tasks/cc/core:base_options",
         "//mediapipe/tasks/cc/core:base_task_api",

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc

@@ -141,16 +141,22 @@ ConvertGestureRecognizerGraphOptionsProto(GestureRecognizerOptions* options) {
   // Configure hand gesture recognizer options.
   auto* hand_gesture_recognizer_graph_options =
       options_proto->mutable_hand_gesture_recognizer_graph_options();
-  if (options->min_gesture_confidence >= 0) {
+  auto canned_gestures_classifier_options_proto =
+      std::make_unique<components::processors::proto::ClassifierOptions>(
+          components::processors::ConvertClassifierOptionsToProto(
+              &(options->canned_gestures_classifier_options)));
   hand_gesture_recognizer_graph_options
       ->mutable_canned_gesture_classifier_graph_options()
       ->mutable_classifier_options()
-        ->set_score_threshold(options->min_gesture_confidence);
+      ->Swap(canned_gestures_classifier_options_proto.get());
+  auto custom_gestures_classifier_options_proto =
+      std::make_unique<components::processors::proto::ClassifierOptions>(
+          components::processors::ConvertClassifierOptionsToProto(
+              &(options->canned_gestures_classifier_options)));
   hand_gesture_recognizer_graph_options
       ->mutable_custom_gesture_classifier_graph_options()
       ->mutable_classifier_options()
-        ->set_score_threshold(options->min_gesture_confidence);
+      ->Swap(canned_gestures_classifier_options_proto.get());
-  }
   return options_proto;
 }
 

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h

@@ -18,12 +18,14 @@ limitations under the License.
 
 #include <memory>
 #include <optional>
+#include <vector>
 
 #include "absl/status/statusor.h"
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/tasks/cc/components/containers/gesture_recognition_result.h"
+#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
 #include "mediapipe/tasks/cc/core/base_options.h"
 #include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
 #include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
@@ -64,12 +66,17 @@ struct GestureRecognizerOptions {
   // successful.
   float min_tracking_confidence = 0.5;
 
-  // The minimum confidence score for the gestures to be considered
+  // TODO  Note this option is subject to change.
-  // successful. If < 0, the gesture confidence thresholds in the model
+  // Options for configuring the canned gestures classifier, such as score
-  // metadata are used.
+  // threshold, allow list and deny list of gestures. The categories for canned
-  // TODO  Note this option is subject to change, after scoring
+  // gesture classifiers are: ["None", "Closed_Fist", "Open_Palm",
-  // merging calculator is implemented.
+  // "Pointing_Up", "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
-  float min_gesture_confidence = -1;
+  components::processors::ClassifierOptions canned_gestures_classifier_options;
+
+  // TODO  Note this option is subject to change.
+  // Options for configuring the custom gestures classifier, such as score
+  // threshold, allow list and deny list of gestures.
+  components::processors::ClassifierOptions custom_gestures_classifier_options;
 
   // The user-defined result callback for processing live stream data.
   // The result callback should only be specified when the running mode is set

mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc

@@ -403,11 +403,11 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
       const core::ModelResources* model_resources,
       const proto::GestureClassifierGraphOptions& options,
       Source<Tensor>& embedding_tensors, Graph& graph) {
-    auto& custom_gesture_classifier_inference = AddInference(
+    auto& gesture_classifier_inference = AddInference(
         *model_resources, options.base_options().acceleration(), graph);
-    embedding_tensors >> custom_gesture_classifier_inference.In(kTensorsTag);
+    embedding_tensors >> gesture_classifier_inference.In(kTensorsTag);
-    auto custom_gesture_inference_out_tensors =
+    auto gesture_inference_out_tensors =
-        custom_gesture_classifier_inference.Out(kTensorsTag);
+        gesture_classifier_inference.Out(kTensorsTag);
     auto& tensors_to_classification =
         graph.AddNode("TensorsToClassificationCalculator");
     MP_RETURN_IF_ERROR(ConfigureTensorsToClassificationCalculator(
@@ -415,8 +415,7 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
         0,
         &tensors_to_classification.GetOptions<
             mediapipe::TensorsToClassificationCalculatorOptions>()));
-    custom_gesture_inference_out_tensors >>
+    gesture_inference_out_tensors >> tensors_to_classification.In(kTensorsTag);
-        tensors_to_classification.In(kTensorsTag);
     return tensors_to_classification.Out("CLASSIFICATIONS")
         .Cast<ClassificationList>();
   }

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD

@@ -137,6 +137,7 @@ android_library(
         "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
         "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
         "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
+        "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
         "//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
         "//third_party:autovalue",
         "@maven//:com_google_guava_guava",

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java

@@ -26,7 +26,7 @@ import com.google.mediapipe.framework.Packet;
 import com.google.mediapipe.framework.PacketGetter;
 import com.google.mediapipe.framework.image.BitmapImageBuilder;
 import com.google.mediapipe.framework.image.MPImage;
-import com.google.mediapipe.tasks.components.processors.proto.ClassifierOptionsProto;
+import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
 import com.google.mediapipe.tasks.core.BaseOptions;
 import com.google.mediapipe.tasks.core.ErrorListener;
 import com.google.mediapipe.tasks.core.OutputHandler;
@@ -398,13 +398,26 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
       public abstract Builder setMinTrackingConfidence(Float value);
 
       /**
-       * Sets the minimum confidence score for the gestures to be considered successful. If < 0, the
+       * Sets the optional {@link ClassifierOptions} controling the canned gestures classifier, such
-       * gesture confidence threshold=0.5 for the model is used.
+       * as score threshold, allow list and deny list of gestures. The categories for canned gesture
+       * classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up", "Thumb_Down",
+       * "Thumb_Up", "Victory", "ILoveYou"]
        *
        * <p>TODO  Note this option is subject to change, after scoring merging
        * calculator is implemented.
        */
-      public abstract Builder setMinGestureConfidence(Float value);
+      public abstract Builder setCannedGesturesClassifierOptions(
+          ClassifierOptions classifierOptions);
+
+      /**
+       * Sets the optional {@link ClassifierOptions} controling the custom gestures classifier, such
+       * as score threshold, allow list and deny list of gestures.
+       *
+       * <p>TODO  Note this option is subject to change, after scoring merging
+       * calculator is implemented.
+       */
+      public abstract Builder setCustomGesturesClassifierOptions(
+          ClassifierOptions classifierOptions);
 
       /**
        * Sets the result listener to receive the detection results asynchronously when the gesture
@@ -454,8 +467,9 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
 
     abstract Optional<Float> minTrackingConfidence();
 
-    // TODO update gesture confidence options after score merging calculator is ready.
+    abstract Optional<ClassifierOptions> cannedGesturesClassifierOptions();
-    abstract Optional<Float> minGestureConfidence();
+
+    abstract Optional<ClassifierOptions> customGesturesClassifierOptions();
 
     abstract Optional<ResultListener<GestureRecognitionResult, MPImage>> resultListener();
 
@@ -467,8 +481,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
           .setNumHands(1)
           .setMinHandDetectionConfidence(0.5f)
           .setMinHandPresenceConfidence(0.5f)
-          .setMinTrackingConfidence(0.5f)
+          .setMinTrackingConfidence(0.5f);
-          .setMinGestureConfidence(-1f);
     }
 
     /**
@@ -511,13 +524,22 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
       HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.Builder
           handGestureRecognizerGraphOptionsBuilder =
               HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder();
-      ClassifierOptionsProto.ClassifierOptions.Builder classifierOptionsBuilder =
+      cannedGesturesClassifierOptions()
-          ClassifierOptionsProto.ClassifierOptions.newBuilder();
+          .ifPresent(
-      minGestureConfidence().ifPresent(classifierOptionsBuilder::setScoreThreshold);
+              classifierOptions -> {
                 handGestureRecognizerGraphOptionsBuilder.setCannedGestureClassifierGraphOptions(
                     GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
-              .setClassifierOptions(classifierOptionsBuilder.build()));
+                        .setClassifierOptions(classifierOptions.convertToProto())
-
+                        .build());
+              });
+      customGesturesClassifierOptions()
+          .ifPresent(
+              classifierOptions -> {
+                handGestureRecognizerGraphOptionsBuilder.setCustomGestureClassifierGraphOptions(
+                    GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
+                        .setClassifierOptions(classifierOptions.convertToProto())
+                        .build());
+              });
       taskOptionsBuilder
           .setHandLandmarkerGraphOptions(handLandmarkerGraphOptionsBuilder.build())
           .setHandGestureRecognizerGraphOptions(handGestureRecognizerGraphOptionsBuilder.build());

mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java

@@ -30,6 +30,7 @@ import com.google.mediapipe.framework.image.MPImage;
 import com.google.mediapipe.tasks.components.containers.Category;
 import com.google.mediapipe.tasks.components.containers.Landmark;
 import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
+import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
 import com.google.mediapipe.tasks.core.BaseOptions;
 import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
 import com.google.mediapipe.tasks.vision.core.RunningMode;
@@ -106,14 +107,15 @@ public class GestureRecognizerTest {
     }
 
     @Test
-    public void recognize_successWithMinGestureConfidence() throws Exception {
+    public void recognize_successWithScoreThreshold() throws Exception {
       GestureRecognizerOptions options =
           GestureRecognizerOptions.builder()
               .setBaseOptions(
                   BaseOptions.builder()
                       .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
                       .build())
-              .setMinGestureConfidence(0.5f)
+              .setCannedGesturesClassifierOptions(
+                  ClassifierOptions.builder().setScoreThreshold(0.5f).build())
               .build();
       GestureRecognizer gestureRecognizer =
           GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
@@ -204,6 +206,113 @@ public class GestureRecognizerTest {
       assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
     }
 
+    @Test
+    public void recognize_successWithAllowGestureFist() throws Exception {
+      GestureRecognizerOptions options =
+          GestureRecognizerOptions.builder()
+              .setBaseOptions(
+                  BaseOptions.builder()
+                      .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
+                      .build())
+              .setNumHands(1)
+              .setCannedGesturesClassifierOptions(
+                  ClassifierOptions.builder()
+                      .setScoreThreshold(0.5f)
+                      .setCategoryAllowlist(Arrays.asList("Closed_Fist"))
+                      .build())
+              .build();
+      GestureRecognizer gestureRecognizer =
+          GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
+      GestureRecognitionResult actualResult =
+          gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
+      GestureRecognitionResult expectedResult =
+          getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
+      assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+    }
+
+    @Test
+    public void recognize_successWithDenyGestureFist() throws Exception {
+      GestureRecognizerOptions options =
+          GestureRecognizerOptions.builder()
+              .setBaseOptions(
+                  BaseOptions.builder()
+                      .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
+                      .build())
+              .setNumHands(1)
+              .setCannedGesturesClassifierOptions(
+                  ClassifierOptions.builder()
+                      .setScoreThreshold(0.5f)
+                      .setCategoryDenylist(Arrays.asList("Closed_Fist"))
+                      .build())
+              .build();
+      GestureRecognizer gestureRecognizer =
+          GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
+      GestureRecognitionResult actualResult =
+          gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
+      assertThat(actualResult.landmarks()).isEmpty();
+      assertThat(actualResult.worldLandmarks()).isEmpty();
+      assertThat(actualResult.handednesses()).isEmpty();
+      assertThat(actualResult.gestures()).isEmpty();
+    }
+
+    @Test
+    public void recognize_successWithAllowAllGestureExceptFist() throws Exception {
+      GestureRecognizerOptions options =
+          GestureRecognizerOptions.builder()
+              .setBaseOptions(
+                  BaseOptions.builder()
+                      .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
+                      .build())
+              .setNumHands(1)
+              .setCannedGesturesClassifierOptions(
+                  ClassifierOptions.builder()
+                      .setScoreThreshold(0.5f)
+                      .setCategoryAllowlist(
+                          Arrays.asList(
+                              "None",
+                              "Open_Palm",
+                              "Pointing_Up",
+                              "Thumb_Down",
+                              "Thumb_Up",
+                              "Victory",
+                              "ILoveYou"))
+                      .build())
+              .build();
+      GestureRecognizer gestureRecognizer =
+          GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
+      GestureRecognitionResult actualResult =
+          gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
+      assertThat(actualResult.landmarks()).isEmpty();
+      assertThat(actualResult.worldLandmarks()).isEmpty();
+      assertThat(actualResult.handednesses()).isEmpty();
+      assertThat(actualResult.gestures()).isEmpty();
+    }
+
+    @Test
+    public void recognize_successWithPreferAlowListThanDenyList() throws Exception {
+      GestureRecognizerOptions options =
+          GestureRecognizerOptions.builder()
+              .setBaseOptions(
+                  BaseOptions.builder()
+                      .setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
+                      .build())
+              .setNumHands(1)
+              .setCannedGesturesClassifierOptions(
+                  ClassifierOptions.builder()
+                      .setScoreThreshold(0.5f)
+                      .setCategoryAllowlist(Arrays.asList("Closed_Fist"))
+                      .setCategoryDenylist(Arrays.asList("Closed_Fist"))
+                      .build())
+              .build();
+      GestureRecognizer gestureRecognizer =
+          GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
+      GestureRecognitionResult actualResult =
+          gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
+      GestureRecognitionResult expectedResult =
+          getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
+      assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+    }
+
     @Test
     public void recognize_failsWithRegionOfInterest() throws Exception {
       GestureRecognizerOptions options =

mediapipe/tasks/python/vision/image_classifier.py

@@ -21,7 +21,6 @@ from mediapipe.python import packet_getter
 # TODO: Import MPImage directly one we have an alias
 from mediapipe.python._framework_bindings import image as image_module
 from mediapipe.python._framework_bindings import packet
-from mediapipe.python._framework_bindings import task_runner
 from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
 from mediapipe.tasks.cc.vision.image_classifier.proto import image_classifier_graph_options_pb2
 from mediapipe.tasks.python.components.containers import classifications
@@ -39,7 +38,6 @@ _ImageClassifierGraphOptionsProto = image_classifier_graph_options_pb2.ImageClas
 _ClassifierOptions = classifier_options.ClassifierOptions
 _RunningMode = vision_task_running_mode.VisionTaskRunningMode
 _TaskInfo = task_info_module.TaskInfo
-_TaskRunner = task_runner.TaskRunner
 
 _CLASSIFICATION_RESULT_OUT_STREAM_NAME = 'classification_result_out'
 _CLASSIFICATION_RESULT_TAG = 'CLASSIFICATION_RESULT'

mediapipe/tasks/python/vision/image_segmenter.py

@@ -21,7 +21,6 @@ from mediapipe.python import packet_creator
 from mediapipe.python import packet_getter
 from mediapipe.python._framework_bindings import image as image_module
 from mediapipe.python._framework_bindings import packet
-from mediapipe.python._framework_bindings import task_runner
 from mediapipe.tasks.cc.components.proto import segmenter_options_pb2
 from mediapipe.tasks.cc.vision.image_segmenter.proto import image_segmenter_options_pb2
 from mediapipe.tasks.python.core import base_options as base_options_module
@@ -35,7 +34,6 @@ _SegmenterOptionsProto = segmenter_options_pb2.SegmenterOptions
 _ImageSegmenterOptionsProto = image_segmenter_options_pb2.ImageSegmenterOptions
 _RunningMode = vision_task_running_mode.VisionTaskRunningMode
 _TaskInfo = task_info_module.TaskInfo
-_TaskRunner = task_runner.TaskRunner
 
 _SEGMENTATION_OUT_STREAM_NAME = 'segmented_mask_out'
 _SEGMENTATION_TAG = 'GROUPED_SEGMENTATION'

mediapipe/tasks/python/vision/object_detector.py

@@ -20,7 +20,6 @@ from mediapipe.python import packet_creator
 from mediapipe.python import packet_getter
 from mediapipe.python._framework_bindings import image as image_module
 from mediapipe.python._framework_bindings import packet as packet_module
-from mediapipe.python._framework_bindings import task_runner as task_runner_module
 from mediapipe.tasks.cc.vision.object_detector.proto import object_detector_options_pb2
 from mediapipe.tasks.python.components.containers import detections as detections_module
 from mediapipe.tasks.python.core import base_options as base_options_module
@@ -33,7 +32,6 @@ _BaseOptions = base_options_module.BaseOptions
 _ObjectDetectorOptionsProto = object_detector_options_pb2.ObjectDetectorOptions
 _RunningMode = running_mode_module.VisionTaskRunningMode
 _TaskInfo = task_info_module.TaskInfo
-_TaskRunner = task_runner_module.TaskRunner
 
 _DETECTIONS_OUT_STREAM_NAME = 'detections_out'
 _DETECTIONS_TAG = 'DETECTIONS'