From c1b460920c0937617baaad737b6d94ab5a378b5b Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 6 Mar 2023 21:58:37 -0800 Subject: [PATCH] normalized keypoint container. Detection adds optional keypoint list. PiperOrigin-RevId: 514624925 --- .../tasks/components/containers/BUILD | 16 ++++ .../components/containers/Detection.java | 26 +++++- .../containers/NormalizedKeypoint.java | 91 +++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedKeypoint.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD index 4d302b950..0ce60d946 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD @@ -38,8 +38,12 @@ android_library( android_library( name = "detection", srcs = ["Detection.java"], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], deps = [ ":category", + ":normalizedkeypoint", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], @@ -92,6 +96,18 @@ android_library( ], ) +android_library( + name = "normalizedkeypoint", + srcs = ["NormalizedKeypoint.java"], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + deps = [ + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) + # Expose the java source files for building mediapipe tasks core AAR. filegroup( name = "java_src", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Detection.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Detection.java index c02c8025a..e2c26cf3d 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Detection.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Detection.java @@ -18,6 +18,7 @@ import android.graphics.RectF; import com.google.auto.value.AutoValue; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Represents one detected object in the results of {@link @@ -39,7 +40,22 @@ public abstract class Detection { // such as Guava, because it may introduce conflicts with clients who also happen to use those // libraries. Therefore, instead of using ImmutableList here, we convert the List into // unmodifiableList - return new AutoValue_Detection(Collections.unmodifiableList(categories), boundingBox); + return new AutoValue_Detection( + Collections.unmodifiableList(categories), boundingBox, Optional.empty()); + } + + /** + * Creates a {@link Detection} instance from a list of {@link Category} and a bounding box. + * + * @param categories a list of {@link Category} objects that contain category name, display name, + * score, and the label index. + * @param boundingBox a {@link RectF} object to represent the bounding box. + * @param keypoints an optional list of {@link NormalizedKeypoints} associated with the detection. + */ + public static Detection create( + List categories, RectF boundingBox, Optional> keypoints) { + return new AutoValue_Detection( + Collections.unmodifiableList(categories), boundingBox, keypoints); } /** A list of {@link Category} objects. */ @@ -47,4 +63,12 @@ public abstract class Detection { /** A {@link RectF} object to represent the bounding box of the detected object. */ public abstract RectF boundingBox(); + + /** + * An optional list of {@link NormalizedKeypoint} associated with the detection. Keypoints + * represent interesting points related to the detection. For example, the keypoints represent the + * eyes, ear and mouth from face detection model. Or in the template matching detection, e.g. + * KNIFT, they can represent the feature points for template matching. + */ + public abstract Optional> keypoints(); } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedKeypoint.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedKeypoint.java new file mode 100644 index 000000000..1a4c1ebab --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedKeypoint.java @@ -0,0 +1,91 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import com.google.auto.value.AutoValue; +import java.util.Objects; +import java.util.Optional; + +/** + * Normalized keypoint represents a point in 2D space with x, y coordinates. x and y are normalized + * to [0.0, 1.0] by the image width and height respectively. + */ +@AutoValue +public abstract class NormalizedKeypoint { + private static final float TOLERANCE = 1e-6f; + + /** + * Creates a {@link NormalizedKeypoint} instance from normalized x and y coordinates. + * + * @param x the x coordinates of the normalized keypoint. + * @param y the y coordinates of the normalized keypoint. + */ + public static NormalizedKeypoint create(float x, float y) { + return new AutoValue_NormalizedKeypoint(x, y, Optional.empty(), Optional.empty()); + } + + /** + * Creates a {@link NormalizedKeypoint} instance from normalized x and y coordinates, and the + * optional label and keypoint score. + * + * @param x the x coordinates of the normalized keypoint. + * @param y the y coordinates of the normalized keypoint. + * @param label optional label of the keypoint. + * @param score optional score of the keypoint. + */ + public static NormalizedKeypoint create( + float x, float y, Optional label, Optional score) { + return new AutoValue_NormalizedKeypoint(x, y, label, score); + } + + // The x coordinates of the normalized keypoint. + public abstract float x(); + + // The y coordinates of the normalized keypoint. + public abstract float y(); + + // optional label of the keypoint. + public abstract Optional label(); + + // optional score of the keypoint. + public abstract Optional score(); + + @Override + public final boolean equals(Object o) { + if (!(o instanceof NormalizedKeypoint)) { + return false; + } + NormalizedKeypoint other = (NormalizedKeypoint) o; + return Math.abs(other.x() - this.x()) < TOLERANCE && Math.abs(other.x() - this.y()) < TOLERANCE; + } + + @Override + public final int hashCode() { + return Objects.hash(x(), y(), label(), score()); + } + + @Override + public final String toString() { + return ""; + } +}