cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'google:master' into gesture-recognizer-python

Browse Source
This commit is contained in:
Kinar R 2022-11-01 08:16:43 +05:30 committed by GitHub
commit 1aaaca1e12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 189 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediapipe/calculators/core/BUILD
View File

@ -222,10 +222,10 @@ cc_library(
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",

4
mediapipe/calculators/core/end_loop_calculator.cc
View File

@ -19,6 +19,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/util/render_data.pb.h"
#include "tensorflow/lite/interpreter.h"
@ -58,4 +59,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
EndLoopDetectionCalculator;
REGISTER_CALCULATOR(EndLoopDetectionCalculator);
typedef EndLoopCalculator<std::vector<Matrix>> EndLoopMatrixCalculator;
REGISTER_CALCULATOR(EndLoopMatrixCalculator);
} // namespace mediapipe

1
mediapipe/tasks/cc/vision/gesture_recognizer/BUILD
View File

@ -141,6 +141,7 @@ cc_library(
"//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/components:image_preprocessing",
"//mediapipe/tasks/cc/components/containers:gesture_recognition_result",
"//mediapipe/tasks/cc/components/processors:classifier_options",
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
"//mediapipe/tasks/cc/core:base_options",
"//mediapipe/tasks/cc/core:base_task_api",

26
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc
View File

@ -141,16 +141,22 @@ ConvertGestureRecognizerGraphOptionsProto(GestureRecognizerOptions* options) {
// Configure hand gesture recognizer options.
auto* hand_gesture_recognizer_graph_options =
options_proto->mutable_hand_gesture_recognizer_graph_options();
if (options->min_gesture_confidence >= 0) {
hand_gesture_recognizer_graph_options
->mutable_canned_gesture_classifier_graph_options()
->mutable_classifier_options()
->set_score_threshold(options->min_gesture_confidence);
hand_gesture_recognizer_graph_options
->mutable_custom_gesture_classifier_graph_options()
->mutable_classifier_options()
->set_score_threshold(options->min_gesture_confidence);
}
auto canned_gestures_classifier_options_proto =
std::make_unique<components::processors::proto::ClassifierOptions>(
components::processors::ConvertClassifierOptionsToProto(
&(options->canned_gestures_classifier_options)));
hand_gesture_recognizer_graph_options
->mutable_canned_gesture_classifier_graph_options()
->mutable_classifier_options()
->Swap(canned_gestures_classifier_options_proto.get());
auto custom_gestures_classifier_options_proto =
std::make_unique<components::processors::proto::ClassifierOptions>(
components::processors::ConvertClassifierOptionsToProto(
&(options->canned_gestures_classifier_options)));
hand_gesture_recognizer_graph_options
->mutable_custom_gesture_classifier_graph_options()
->mutable_classifier_options()
->Swap(canned_gestures_classifier_options_proto.get());
return options_proto;
}

19
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h
View File

@ -18,12 +18,14 @@ limitations under the License.
#include <memory>
#include <optional>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/tasks/cc/components/containers/gesture_recognition_result.h"
#include "mediapipe/tasks/cc/components/processors/classifier_options.h"
#include "mediapipe/tasks/cc/core/base_options.h"
#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h"
#include "mediapipe/tasks/cc/vision/core/image_processing_options.h"
@ -64,12 +66,17 @@ struct GestureRecognizerOptions {
// successful.
float min_tracking_confidence = 0.5;
// The minimum confidence score for the gestures to be considered
// successful. If < 0, the gesture confidence thresholds in the model
// metadata are used.
// TODO Note this option is subject to change, after scoring
// merging calculator is implemented.
float min_gesture_confidence = -1;
// TODO Note this option is subject to change.
// Options for configuring the canned gestures classifier, such as score
// threshold, allow list and deny list of gestures. The categories for canned
// gesture classifiers are: ["None", "Closed_Fist", "Open_Palm",
// "Pointing_Up", "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
components::processors::ClassifierOptions canned_gestures_classifier_options;
// TODO Note this option is subject to change.
// Options for configuring the custom gestures classifier, such as score
// threshold, allow list and deny list of gestures.
components::processors::ClassifierOptions custom_gestures_classifier_options;
// The user-defined result callback for processing live stream data.
// The result callback should only be specified when the running mode is set

11
mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc
View File

@ -403,11 +403,11 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
const core::ModelResources* model_resources,
const proto::GestureClassifierGraphOptions& options,
Source<Tensor>& embedding_tensors, Graph& graph) {
auto& custom_gesture_classifier_inference = AddInference(
auto& gesture_classifier_inference = AddInference(
*model_resources, options.base_options().acceleration(), graph);
embedding_tensors >> custom_gesture_classifier_inference.In(kTensorsTag);
auto custom_gesture_inference_out_tensors =
custom_gesture_classifier_inference.Out(kTensorsTag);
embedding_tensors >> gesture_classifier_inference.In(kTensorsTag);
auto gesture_inference_out_tensors =
gesture_classifier_inference.Out(kTensorsTag);
auto& tensors_to_classification =
graph.AddNode("TensorsToClassificationCalculator");
MP_RETURN_IF_ERROR(ConfigureTensorsToClassificationCalculator(
@ -415,8 +415,7 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph {
0,
&tensors_to_classification.GetOptions<
mediapipe::TensorsToClassificationCalculatorOptions>()));
custom_gesture_inference_out_tensors >>
tensors_to_classification.In(kTensorsTag);
gesture_inference_out_tensors >> tensors_to_classification.In(kTensorsTag);
return tensors_to_classification.Out("CLASSIFICATIONS")
.Cast<ClassificationList>();
}

1
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD
View File

@ -137,6 +137,7 @@ android_library(
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue",
"@maven//:com_google_guava_guava",

52
mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java
View File

@ -26,7 +26,7 @@ import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.processors.proto.ClassifierOptionsProto;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.core.ErrorListener;
import com.google.mediapipe.tasks.core.OutputHandler;
@ -398,13 +398,26 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
public abstract Builder setMinTrackingConfidence(Float value);
/**
* Sets the minimum confidence score for the gestures to be considered successful. If < 0, the
* gesture confidence threshold=0.5 for the model is used.
* Sets the optional {@link ClassifierOptions} controling the canned gestures classifier, such
* as score threshold, allow list and deny list of gestures. The categories for canned gesture
* classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up", "Thumb_Down",
* "Thumb_Up", "Victory", "ILoveYou"]
*
* <p>TODO Note this option is subject to change, after scoring merging
* calculator is implemented.
*/
public abstract Builder setMinGestureConfidence(Float value);
public abstract Builder setCannedGesturesClassifierOptions(
ClassifierOptions classifierOptions);
/**
* Sets the optional {@link ClassifierOptions} controling the custom gestures classifier, such
* as score threshold, allow list and deny list of gestures.
*
* <p>TODO Note this option is subject to change, after scoring merging
* calculator is implemented.
*/
public abstract Builder setCustomGesturesClassifierOptions(
ClassifierOptions classifierOptions);
/**
* Sets the result listener to receive the detection results asynchronously when the gesture
@ -454,8 +467,9 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
abstract Optional<Float> minTrackingConfidence();
// TODO update gesture confidence options after score merging calculator is ready.
abstract Optional<Float> minGestureConfidence();
abstract Optional<ClassifierOptions> cannedGesturesClassifierOptions();
abstract Optional<ClassifierOptions> customGesturesClassifierOptions();
abstract Optional<ResultListener<GestureRecognitionResult, MPImage>> resultListener();
@ -467,8 +481,7 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
.setNumHands(1)
.setMinHandDetectionConfidence(0.5f)
.setMinHandPresenceConfidence(0.5f)
.setMinTrackingConfidence(0.5f)
.setMinGestureConfidence(-1f);
.setMinTrackingConfidence(0.5f);
}
/**
@ -511,13 +524,22 @@ public final class GestureRecognizer extends BaseVisionTaskApi {
HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.Builder
handGestureRecognizerGraphOptionsBuilder =
HandGestureRecognizerGraphOptionsProto.HandGestureRecognizerGraphOptions.newBuilder();
ClassifierOptionsProto.ClassifierOptions.Builder classifierOptionsBuilder =
ClassifierOptionsProto.ClassifierOptions.newBuilder();
minGestureConfidence().ifPresent(classifierOptionsBuilder::setScoreThreshold);
handGestureRecognizerGraphOptionsBuilder.setCannedGestureClassifierGraphOptions(
GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
.setClassifierOptions(classifierOptionsBuilder.build()));
cannedGesturesClassifierOptions()
.ifPresent(
classifierOptions -> {
handGestureRecognizerGraphOptionsBuilder.setCannedGestureClassifierGraphOptions(
GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
.setClassifierOptions(classifierOptions.convertToProto())
.build());
});
customGesturesClassifierOptions()
.ifPresent(
classifierOptions -> {
handGestureRecognizerGraphOptionsBuilder.setCustomGestureClassifierGraphOptions(
GestureClassifierGraphOptionsProto.GestureClassifierGraphOptions.newBuilder()
.setClassifierOptions(classifierOptions.convertToProto())
.build());
});
taskOptionsBuilder
.setHandLandmarkerGraphOptions(handLandmarkerGraphOptionsBuilder.build())
.setHandGestureRecognizerGraphOptions(handGestureRecognizerGraphOptionsBuilder.build());

113
mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java
View File

@ -30,6 +30,7 @@ import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark;
import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
import com.google.mediapipe.tasks.vision.core.RunningMode;
@ -106,14 +107,15 @@ public class GestureRecognizerTest {
}
@Test
public void recognize_successWithMinGestureConfidence() throws Exception {
public void recognize_successWithScoreThreshold() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setMinGestureConfidence(0.5f)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder().setScoreThreshold(0.5f).build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
@ -204,6 +206,113 @@ public class GestureRecognizerTest {
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_successWithAllowGestureFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_successWithDenyGestureFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryDenylist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
assertThat(actualResult.landmarks()).isEmpty();
assertThat(actualResult.worldLandmarks()).isEmpty();
assertThat(actualResult.handednesses()).isEmpty();
assertThat(actualResult.gestures()).isEmpty();
}
@Test
public void recognize_successWithAllowAllGestureExceptFist() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(
Arrays.asList(
"None",
"Open_Palm",
"Pointing_Up",
"Thumb_Down",
"Thumb_Up",
"Victory",
"ILoveYou"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
assertThat(actualResult.landmarks()).isEmpty();
assertThat(actualResult.worldLandmarks()).isEmpty();
assertThat(actualResult.handednesses()).isEmpty();
assertThat(actualResult.gestures()).isEmpty();
}
@Test
public void recognize_successWithPreferAlowListThanDenyList() throws Exception {
GestureRecognizerOptions options =
GestureRecognizerOptions.builder()
.setBaseOptions(
BaseOptions.builder()
.setModelAssetPath(GESTURE_RECOGNIZER_BUNDLE_ASSET_FILE)
.build())
.setNumHands(1)
.setCannedGesturesClassifierOptions(
ClassifierOptions.builder()
.setScoreThreshold(0.5f)
.setCategoryAllowlist(Arrays.asList("Closed_Fist"))
.setCategoryDenylist(Arrays.asList("Closed_Fist"))
.build())
.build();
GestureRecognizer gestureRecognizer =
GestureRecognizer.createFromOptions(ApplicationProvider.getApplicationContext(), options);
GestureRecognitionResult actualResult =
gestureRecognizer.recognize(getImageFromAsset(FIST_IMAGE));
GestureRecognitionResult expectedResult =
getExpectedGestureRecognitionResult(FIST_LANDMARKS, FIST_LABEL);
assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
}
@Test
public void recognize_failsWithRegionOfInterest() throws Exception {
GestureRecognizerOptions options =

2
mediapipe/tasks/python/vision/image_classifier.py
View File

@ -21,7 +21,6 @@ from mediapipe.python import packet_getter
# TODO: Import MPImage directly one we have an alias
from mediapipe.python._framework_bindings import image as image_module
from mediapipe.python._framework_bindings import packet
from mediapipe.python._framework_bindings import task_runner
from mediapipe.tasks.cc.components.containers.proto import classifications_pb2
from mediapipe.tasks.cc.vision.image_classifier.proto import image_classifier_graph_options_pb2
from mediapipe.tasks.python.components.containers import classifications
@ -39,7 +38,6 @@ _ImageClassifierGraphOptionsProto = image_classifier_graph_options_pb2.ImageClas
_ClassifierOptions = classifier_options.ClassifierOptions
_RunningMode = vision_task_running_mode.VisionTaskRunningMode
_TaskInfo = task_info_module.TaskInfo
_TaskRunner = task_runner.TaskRunner
_CLASSIFICATION_RESULT_OUT_STREAM_NAME = 'classification_result_out'
_CLASSIFICATION_RESULT_TAG = 'CLASSIFICATION_RESULT'

2
mediapipe/tasks/python/vision/image_segmenter.py
View File

@ -21,7 +21,6 @@ from mediapipe.python import packet_creator
from mediapipe.python import packet_getter
from mediapipe.python._framework_bindings import image as image_module
from mediapipe.python._framework_bindings import packet
from mediapipe.python._framework_bindings import task_runner
from mediapipe.tasks.cc.components.proto import segmenter_options_pb2
from mediapipe.tasks.cc.vision.image_segmenter.proto import image_segmenter_options_pb2
from mediapipe.tasks.python.core import base_options as base_options_module
@ -35,7 +34,6 @@ _SegmenterOptionsProto = segmenter_options_pb2.SegmenterOptions
_ImageSegmenterOptionsProto = image_segmenter_options_pb2.ImageSegmenterOptions
_RunningMode = vision_task_running_mode.VisionTaskRunningMode
_TaskInfo = task_info_module.TaskInfo
_TaskRunner = task_runner.TaskRunner
_SEGMENTATION_OUT_STREAM_NAME = 'segmented_mask_out'
_SEGMENTATION_TAG = 'GROUPED_SEGMENTATION'

2
mediapipe/tasks/python/vision/object_detector.py
View File

@ -20,7 +20,6 @@ from mediapipe.python import packet_creator
from mediapipe.python import packet_getter
from mediapipe.python._framework_bindings import image as image_module
from mediapipe.python._framework_bindings import packet as packet_module
from mediapipe.python._framework_bindings import task_runner as task_runner_module
from mediapipe.tasks.cc.vision.object_detector.proto import object_detector_options_pb2
from mediapipe.tasks.python.components.containers import detections as detections_module
from mediapipe.tasks.python.core import base_options as base_options_module
@ -33,7 +32,6 @@ _BaseOptions = base_options_module.BaseOptions
_ObjectDetectorOptionsProto = object_detector_options_pb2.ObjectDetectorOptions
_RunningMode = running_mode_module.VisionTaskRunningMode
_TaskInfo = task_info_module.TaskInfo
_TaskRunner = task_runner_module.TaskRunner
_DETECTIONS_OUT_STREAM_NAME = 'detections_out'
_DETECTIONS_TAG = 'DETECTIONS'