diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.proto b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.proto index dee102244..2afbd507b 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.proto +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.proto @@ -22,6 +22,9 @@ import "mediapipe/tasks/cc/core/proto/base_options.proto"; import "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.proto"; import "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options.proto"; +option java_package = "com.google.mediapipe.tasks.vision.gesturerecognizer"; +option java_outer_classname = "GestureRecognizerGraphOptionsProto"; + message GestureRecognizerGraphOptions { extend mediapipe.CalculatorOptions { optional GestureRecognizerGraphOptions ext = 479097054; diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD index 23e6124cc..610bec911 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD @@ -54,3 +54,12 @@ android_library( "@maven//:com_google_guava_guava", ], ) + +android_library( + name = "landmark", + srcs = ["Landmark.java"], + deps = [ + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java new file mode 100644 index 000000000..3f96d7779 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java @@ -0,0 +1,66 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import com.google.auto.value.AutoValue; +import java.util.Objects; + +/** + * Landmark represents a point in 3D space with x, y, z coordinates. If normalized is true, the + * landmark coordinates is normalized respect to the dimension of image, and the coordinates values + * are in the range of [0,1]. Otherwise, it represenet a point in world coordinates. + */ +@AutoValue +public abstract class Landmark { + private static final float TOLERANCE = 1e-6f; + + public static Landmark create(float x, float y, float z, boolean normalized) { + return new AutoValue_Landmark(x, y, z, normalized); + } + + // The x coordniates of the landmark. + public abstract float x(); + + // The y coordniates of the landmark. + public abstract float y(); + + // The z coordniates of the landmark. + public abstract float z(); + + // Whether this landmark is normalized with respect to the image size. + public abstract boolean normalized(); + + @Override + public final boolean equals(Object o) { + if (!(o instanceof Landmark)) { + return false; + } + Landmark other = (Landmark) o; + return other.normalized() == this.normalized() + && Math.abs(other.x() - this.x()) < TOLERANCE + && Math.abs(other.x() - this.y()) < TOLERANCE + && Math.abs(other.x() - this.z()) < TOLERANCE; + } + + @Override + public final int hashCode() { + return Objects.hash(x(), y(), z(), normalized()); + } + + @Override + public final String toString() { + return ""; + } +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/AndroidManifest.xml b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/AndroidManifest.xml new file mode 100644 index 000000000..38f98f1a1 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/BUILD new file mode 100644 index 000000000..eb3eca52b --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/BUILD @@ -0,0 +1,40 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +android_library( + name = "gesturerecognizer", + srcs = [ + "GestureRecognitionResult.java", + ], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = ":AndroidManifest.xml", + deps = [ + "//mediapipe/framework:calculator_options_java_proto_lite", + "//mediapipe/framework/formats:classification_java_proto_lite", + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognitionResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognitionResult.java new file mode 100644 index 000000000..fd764cb18 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognitionResult.java @@ -0,0 +1,128 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.gesturerecognizer; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.LandmarkProto.Landmark; +import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.Classification; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import com.google.mediapipe.tasks.components.containers.Category; +import com.google.mediapipe.tasks.core.TaskResult; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Represents the gesture recognition results generated by {@link GestureRecognizer}. */ +@AutoValue +public abstract class GestureRecognitionResult implements TaskResult { + + /** + * Creates a {@link GestureRecognitionResult} instance from the lists of landmarks, handedness, + * and gestures protobuf messages. + * + * @param landmarksProto a List of {@link NormalizedLandmarkList} + * @param worldLandmarksProto a List of {@link LandmarkList} + * @param handednessesProto a List of {@link ClassificationList} + * @param gesturesProto a List of {@link ClassificationList} + */ + static GestureRecognitionResult create( + List landmarksProto, + List worldLandmarksProto, + List handednessesProto, + List gesturesProto, + long timestampMs) { + List> multiHandLandmarks = + new ArrayList<>(); + List> multiHandWorldLandmarks = + new ArrayList<>(); + List> multiHandHandednesses = new ArrayList<>(); + List> multiHandGestures = new ArrayList<>(); + for (NormalizedLandmarkList handLandmarksProto : landmarksProto) { + List handLandmarks = + new ArrayList<>(); + multiHandLandmarks.add(handLandmarks); + for (NormalizedLandmark handLandmarkProto : handLandmarksProto.getLandmarkList()) { + handLandmarks.add( + com.google.mediapipe.tasks.components.containers.Landmark.create( + handLandmarkProto.getX(), + handLandmarkProto.getY(), + handLandmarkProto.getZ(), + true)); + } + } + for (LandmarkList handWorldLandmarksProto : worldLandmarksProto) { + List handWorldLandmarks = + new ArrayList<>(); + multiHandWorldLandmarks.add(handWorldLandmarks); + for (Landmark handWorldLandmarkProto : handWorldLandmarksProto.getLandmarkList()) { + handWorldLandmarks.add( + com.google.mediapipe.tasks.components.containers.Landmark.create( + handWorldLandmarkProto.getX(), + handWorldLandmarkProto.getY(), + handWorldLandmarkProto.getZ(), + false)); + } + } + for (ClassificationList handednessProto : handednessesProto) { + List handedness = new ArrayList<>(); + multiHandHandednesses.add(handedness); + for (Classification classification : handednessProto.getClassificationList()) { + handedness.add( + Category.create( + classification.getScore(), + classification.getIndex(), + classification.getLabel(), + classification.getDisplayName())); + } + } + for (ClassificationList gestureProto : gesturesProto) { + List gestures = new ArrayList<>(); + multiHandGestures.add(gestures); + for (Classification classification : gestureProto.getClassificationList()) { + gestures.add( + Category.create( + classification.getScore(), + classification.getIndex(), + classification.getLabel(), + classification.getDisplayName())); + } + } + return new AutoValue_GestureRecognitionResult( + timestampMs, + Collections.unmodifiableList(multiHandLandmarks), + Collections.unmodifiableList(multiHandWorldLandmarks), + Collections.unmodifiableList(multiHandHandednesses), + Collections.unmodifiableList(multiHandGestures)); + } + + @Override + public abstract long timestampMs(); + + /** Hand landmarks of detected hands. */ + public abstract List> landmarks(); + + /** Hand landmarks in world coordniates of detected hands. */ + public abstract List> + worldLandmarks(); + + /** Handedness of detected hands. */ + public abstract List> handednesses(); + + /** Recognized hand gestures of detected hands */ + public abstract List> gestures(); +}