builder()
+ .setTaskName(HolisticLandmarker.class.getSimpleName())
+ .setTaskRunningModeName(landmarkerOptions.runningMode().name())
+ .setTaskGraphName(TASK_GRAPH_NAME)
+ .setInputStreams(INPUT_STREAMS)
+ .setOutputStreams(outputStreams)
+ .setTaskOptions(landmarkerOptions)
+ .setEnableFlowLimiting(landmarkerOptions.runningMode() == RunningMode.LIVE_STREAM)
+ .build(),
+ handler);
+ return new HolisticLandmarker(runner, landmarkerOptions.runningMode());
+ }
+
+ /**
+ * Constructor to initialize an {@link HolisticLandmarker} from a {@link TaskRunner} and a {@link
+ * RunningMode}.
+ *
+ * @param taskRunner a {@link TaskRunner}.
+ * @param runningMode a mediapipe vision task {@link RunningMode}.
+ */
+ private HolisticLandmarker(TaskRunner taskRunner, RunningMode runningMode) {
+ super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, /* normRectStreamName= */ "");
+ }
+
+ /**
+ * Performs holistic landmarks detection on the provided single image with default image
+ * processing options, i.e. without any rotation applied. Only use this method when the {@link
+ * HolisticLandmarker} is created with {@link RunningMode.IMAGE}.
+ *
+ * {@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public HolisticLandmarkerResult detect(MPImage image) {
+ return detect(image, ImageProcessingOptions.builder().build());
+ }
+
+ /**
+ * Performs holistic landmarks detection on the provided single image. Only use this method when
+ * the {@link HolisticLandmarker} is created with {@link RunningMode.IMAGE}.
+ *
+ * {@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference. Note that region-of-interest is not supported
+ * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
+ * this method throwing an IllegalArgumentException.
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public HolisticLandmarkerResult detect(
+ MPImage image, ImageProcessingOptions imageProcessingOptions) {
+ validateImageProcessingOptions(imageProcessingOptions);
+ return (HolisticLandmarkerResult) processImageData(image, imageProcessingOptions);
+ }
+
+ /**
+ * Performs holistic landmarks detection on the provided video frame with default image processing
+ * options, i.e. without any rotation applied. Only use this method when the {@link
+ * HolisticLandmarker} is created with {@link RunningMode.VIDEO}.
+ *
+ * It's required to provide the video frame"s timestamp (in milliseconds). The input timestamps
+ * must be monotonically increasing.
+ *
+ *
{@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param timestampMs the input timestamp (in milliseconds).
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public HolisticLandmarkerResult detectForVideo(MPImage image, long timestampMs) {
+ return detectForVideo(image, ImageProcessingOptions.builder().build(), timestampMs);
+ }
+
+ /**
+ * Performs holistic landmarks detection on the provided video frame. Only use this method when
+ * the {@link HolisticLandmarker} is created with {@link RunningMode.VIDEO}.
+ *
+ * It's required to provide the video frame"s timestamp (in milliseconds). The input timestamps
+ * must be monotonically increasing.
+ *
+ *
{@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference. Note that region-of-interest is not supported
+ * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
+ * this method throwing an IllegalArgumentException.
+ * @param timestampMs the input timestamp (in milliseconds).
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public HolisticLandmarkerResult detectForVideo(
+ MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
+ validateImageProcessingOptions(imageProcessingOptions);
+ return (HolisticLandmarkerResult) processVideoData(image, imageProcessingOptions, timestampMs);
+ }
+
+ /**
+ * Sends live image data to perform holistic landmarks detection with default image processing
+ * options, i.e. without any rotation applied, and the results will be available via the {@link
+ * ResultListener} provided in the {@link HolisticLandmarkerOptions}. Only use this method when
+ * the {@link HolisticLandmarker } is created with {@link RunningMode.LIVE_STREAM}.
+ *
+ * It's required to provide a timestamp (in milliseconds) to indicate when the input image is
+ * sent to the holistic landmarker. The input timestamps must be monotonically increasing.
+ *
+ *
{@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param timestampMs the input timestamp (in milliseconds).
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public void detectAsync(MPImage image, long timestampMs) {
+ detectAsync(image, ImageProcessingOptions.builder().build(), timestampMs);
+ }
+
+ /**
+ * Sends live image data to perform holistic landmarks detection, and the results will be
+ * available via the {@link ResultListener} provided in the {@link HolisticLandmarkerOptions}.
+ * Only use this method when the {@link HolisticLandmarker} is created with {@link
+ * RunningMode.LIVE_STREAM}.
+ *
+ * It's required to provide a timestamp (in milliseconds) to indicate when the input image is
+ * sent to the holistic landmarker. The input timestamps must be monotonically increasing.
+ *
+ *
{@link HolisticLandmarker} supports the following color space types:
+ *
+ *
+ * - {@link Bitmap.Config.ARGB_8888}
+ *
+ *
+ * @param image a MediaPipe {@link MPImage} object for processing.
+ * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the
+ * input image before running inference. Note that region-of-interest is not supported
+ * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in
+ * this method throwing an IllegalArgumentException.
+ * @param timestampMs the input timestamp (in milliseconds).
+ * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a
+ * region-of-interest.
+ * @throws MediaPipeException if there is an internal error.
+ */
+ public void detectAsync(
+ MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) {
+ validateImageProcessingOptions(imageProcessingOptions);
+ sendLiveStreamData(image, imageProcessingOptions, timestampMs);
+ }
+
+ /** Options for setting up an {@link HolisticLandmarker}. */
+ @AutoValue
+ public abstract static class HolisticLandmarkerOptions extends TaskOptions {
+
+ /** Builder for {@link HolisticLandmarkerOptions}. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ /** Sets the base options for the holistic landmarker task. */
+ public abstract Builder setBaseOptions(BaseOptions value);
+
+ /**
+ * Sets the running mode for the holistic landmarker task. Defaults to the image mode.
+ * Holistic landmarker has three modes:
+ *
+ *
+ * - IMAGE: The mode for detecting holistic landmarks on single image inputs.
+ *
- VIDEO: The mode for detecting holistic landmarks on the decoded frames of a video.
+ *
- LIVE_STREAM: The mode for for detecting holistic landmarks on a live stream of input
+ * data, such as from camera. In this mode, {@code setResultListener} must be called to
+ * set up a listener to receive the detection results asynchronously.
+ *
+ */
+ public abstract Builder setRunningMode(RunningMode value);
+
+ /**
+ * Sets minimum confidence score for the face detection to be considered successful. Defaults
+ * to 0.5.
+ */
+ public abstract Builder setMinFaceDetectionConfidence(Float value);
+
+ /**
+ * The minimum threshold for the face suppression score in the face detection. Defaults to
+ * 0.3.
+ */
+ public abstract Builder setMinFaceSuppressionThreshold(Float value);
+
+ /**
+ * Sets minimum confidence score for the face landmark detection to be considered successful.
+ * Defaults to 0.5.
+ */
+ public abstract Builder setMinFaceLandmarksConfidence(Float value);
+
+ /**
+ * The minimum confidence score for the pose detection to be considered successful. Defaults
+ * to 0.5.
+ */
+ public abstract Builder setMinPoseDetectionConfidence(Float value);
+
+ /**
+ * The minimum threshold for the pose suppression score in the pose detection. Defaults to
+ * 0.3.
+ */
+ public abstract Builder setMinPoseSuppressionThreshold(Float value);
+
+ /**
+ * The minimum confidence score for the pose landmarks detection to be considered successful.
+ * Defaults to 0.5.
+ */
+ public abstract Builder setMinPoseLandmarksConfidence(Float value);
+
+ /**
+ * The minimum confidence score for the hand landmark detection to be considered successful.
+ * Defaults to 0.5.
+ */
+ public abstract Builder setMinHandLandmarksConfidence(Float value);
+
+ /** Whether to output segmentation masks. Defaults to false. */
+ public abstract Builder setOutputPoseSegmentationMasks(Boolean value);
+
+ /** Whether to output face blendshapes. Defaults to false. */
+ public abstract Builder setOutputFaceBlendshapes(Boolean value);
+
+ /**
+ * Sets the result listener to receive the detection results asynchronously when the holistic
+ * landmarker is in the live stream mode.
+ */
+ public abstract Builder setResultListener(
+ ResultListener value);
+
+ /** Sets an optional error listener. */
+ public abstract Builder setErrorListener(ErrorListener value);
+
+ abstract HolisticLandmarkerOptions autoBuild();
+
+ /**
+ * Validates and builds the {@link HolisticLandmarkerOptions} instance.
+ *
+ * @throws IllegalArgumentException if the result listener and the running mode are not
+ * properly configured. The result listener should only be set when the holistic
+ * landmarker is in the live stream mode.
+ */
+ public final HolisticLandmarkerOptions build() {
+ HolisticLandmarkerOptions options = autoBuild();
+ if (options.runningMode() == RunningMode.LIVE_STREAM) {
+ if (!options.resultListener().isPresent()) {
+ throw new IllegalArgumentException(
+ "The holistic landmarker is in the live stream mode, a user-defined result listener"
+ + " must be provided in HolisticLandmarkerOptions.");
+ }
+ } else if (options.resultListener().isPresent()) {
+ throw new IllegalArgumentException(
+ "The holistic landmarker is in the image or the video mode, a user-defined result"
+ + " listener shouldn't be provided in HolisticLandmarkerOptions.");
+ }
+ return options;
+ }
+ }
+
+ abstract BaseOptions baseOptions();
+
+ abstract RunningMode runningMode();
+
+ abstract Optional minFaceDetectionConfidence();
+
+ abstract Optional minFaceSuppressionThreshold();
+
+ abstract Optional minFaceLandmarksConfidence();
+
+ abstract Optional minPoseDetectionConfidence();
+
+ abstract Optional minPoseSuppressionThreshold();
+
+ abstract Optional minPoseLandmarksConfidence();
+
+ abstract Optional minHandLandmarksConfidence();
+
+ abstract Boolean outputFaceBlendshapes();
+
+ abstract Boolean outputPoseSegmentationMasks();
+
+ abstract Optional> resultListener();
+
+ abstract Optional errorListener();
+
+ public static Builder builder() {
+ return new AutoValue_HolisticLandmarker_HolisticLandmarkerOptions.Builder()
+ .setRunningMode(RunningMode.IMAGE)
+ .setMinFaceDetectionConfidence(DEFAULT_PRESENCE_THRESHOLD)
+ .setMinFaceSuppressionThreshold(DEFAULT_SUPPRESION_THRESHOLD)
+ .setMinFaceLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD)
+ .setMinPoseDetectionConfidence(DEFAULT_PRESENCE_THRESHOLD)
+ .setMinPoseSuppressionThreshold(DEFAULT_SUPPRESION_THRESHOLD)
+ .setMinPoseLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD)
+ .setMinHandLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD)
+ .setOutputFaceBlendshapes(DEFAULT_OUTPUT_FACE_BLENDSHAPES)
+ .setOutputPoseSegmentationMasks(DEFAULT_OUTPUT_SEGMENTATION_MASKS);
+ }
+
+ /** Converts a {@link HolisticLandmarkerOptions} to a {@link Any} protobuf message. */
+ @Override
+ public Any convertToAnyProto() {
+ HolisticLandmarkerGraphOptions.Builder holisticLandmarkerGraphOptions =
+ HolisticLandmarkerGraphOptions.newBuilder()
+ .setBaseOptions(
+ BaseOptionsProto.BaseOptions.newBuilder()
+ .setUseStreamMode(runningMode() != RunningMode.IMAGE)
+ .mergeFrom(convertBaseOptionsToProto(baseOptions()))
+ .build());
+
+ HandLandmarksDetectorGraphOptions.Builder handLandmarksDetectorGraphOptions =
+ HandLandmarksDetectorGraphOptions.newBuilder();
+ FaceDetectorGraphOptions.Builder faceDetectorGraphOptions =
+ FaceDetectorGraphOptions.newBuilder();
+ FaceLandmarksDetectorGraphOptions.Builder faceLandmarksDetectorGraphOptions =
+ FaceLandmarksDetectorGraphOptions.newBuilder();
+ PoseDetectorGraphOptions.Builder poseDetectorGraphOptions =
+ PoseDetectorGraphOptions.newBuilder();
+ PoseLandmarksDetectorGraphOptions.Builder poseLandmarkerGraphOptions =
+ PoseLandmarksDetectorGraphOptions.newBuilder();
+
+ // Configure hand detector options.
+ minHandLandmarksConfidence()
+ .ifPresent(handLandmarksDetectorGraphOptions::setMinDetectionConfidence);
+
+ // Configure pose detector options.
+ minPoseDetectionConfidence().ifPresent(poseDetectorGraphOptions::setMinDetectionConfidence);
+ minPoseSuppressionThreshold().ifPresent(poseDetectorGraphOptions::setMinSuppressionThreshold);
+ minPoseLandmarksConfidence().ifPresent(poseLandmarkerGraphOptions::setMinDetectionConfidence);
+
+ // Configure face detector options.
+ minFaceDetectionConfidence().ifPresent(faceDetectorGraphOptions::setMinDetectionConfidence);
+ minFaceSuppressionThreshold().ifPresent(faceDetectorGraphOptions::setMinSuppressionThreshold);
+ minFaceLandmarksConfidence()
+ .ifPresent(faceLandmarksDetectorGraphOptions::setMinDetectionConfidence);
+
+ holisticLandmarkerGraphOptions
+ .setHandLandmarksDetectorGraphOptions(handLandmarksDetectorGraphOptions.build())
+ .setFaceDetectorGraphOptions(faceDetectorGraphOptions.build())
+ .setFaceLandmarksDetectorGraphOptions(faceLandmarksDetectorGraphOptions.build())
+ .setPoseDetectorGraphOptions(poseDetectorGraphOptions.build())
+ .setPoseLandmarksDetectorGraphOptions(poseLandmarkerGraphOptions.build());
+
+ return Any.newBuilder()
+ .setTypeUrl(
+ "type.googleapis.com/mediapipe.tasks.vision.holistic_landmarker.proto.HolisticLandmarkerGraphOptions")
+ .setValue(holisticLandmarkerGraphOptions.build().toByteString())
+ .build();
+ }
+ }
+
+ /**
+ * Validates that the provided {@link ImageProcessingOptions} doesn"t contain a
+ * region-of-interest.
+ */
+ private static void validateImageProcessingOptions(
+ ImageProcessingOptions imageProcessingOptions) {
+ if (imageProcessingOptions.regionOfInterest().isPresent()) {
+ throw new IllegalArgumentException("HolisticLandmarker doesn't support region-of-interest.");
+ }
+ }
+
+ private static MPImage getSegmentationMask(List packets, int packetIndex) {
+ int width = PacketGetter.getImageWidth(packets.get(packetIndex));
+ int height = PacketGetter.getImageHeight(packets.get(packetIndex));
+ ByteBuffer buffer = ByteBuffer.allocateDirect(width * height * 4);
+
+ if (!PacketGetter.getImageData(packets.get(packetIndex), buffer)) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.INTERNAL.ordinal(),
+ "There was an error getting the sefmentation mask.");
+ }
+
+ ByteBufferImageBuilder builder =
+ new ByteBufferImageBuilder(buffer, width, height, MPImage.IMAGE_FORMAT_VEC32F1);
+ return builder.build();
+ }
+}
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml
new file mode 100644
index 000000000..22b19b702
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD
new file mode 100644
index 000000000..287602c85
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD
@@ -0,0 +1,19 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+# TODO: Enable this in OSS
diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java
new file mode 100644
index 000000000..f8c87c798
--- /dev/null
+++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java
@@ -0,0 +1,512 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.tasks.vision.holisticlandmarker;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import android.content.res.AssetManager;
+import android.graphics.BitmapFactory;
+import android.graphics.RectF;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.truth.Correspondence;
+import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList;
+import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.image.BitmapImageBuilder;
+import com.google.mediapipe.framework.image.ByteBufferImageBuilder;
+import com.google.mediapipe.framework.image.MPImage;
+import com.google.mediapipe.tasks.components.containers.Category;
+import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
+import com.google.mediapipe.tasks.core.BaseOptions;
+import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
+import com.google.mediapipe.tasks.vision.core.RunningMode;
+import com.google.mediapipe.tasks.vision.holisticlandmarker.HolisticLandmarker.HolisticLandmarkerOptions;
+import com.google.mediapipe.tasks.vision.holisticlandmarker.HolisticResultProto.HolisticResult;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Optional;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+/** Test for {@link HolisticLandmarker}. */
+@RunWith(Suite.class)
+@SuiteClasses({HolisticLandmarkerTest.General.class, HolisticLandmarkerTest.RunningModeTest.class})
+public class HolisticLandmarkerTest {
+ private static final String HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE = "holistic_landmarker.task";
+ private static final String POSE_IMAGE = "male_full_height_hands.jpg";
+ private static final String CAT_IMAGE = "cat.jpg";
+ private static final String HOLISTIC_RESULT = "male_full_height_hands_result_cpu.pb";
+ private static final String TAG = "Holistic Landmarker Test";
+ private static final float FACE_LANDMARKS_ERROR_TOLERANCE = 0.03f;
+ private static final float FACE_BLENDSHAPES_ERROR_TOLERANCE = 0.13f;
+ private static final MPImage PLACEHOLDER_MASK =
+ new ByteBufferImageBuilder(
+ ByteBuffer.allocate(0), /* widht= */ 0, /* height= */ 0, MPImage.IMAGE_FORMAT_VEC32F1)
+ .build();
+ private static final int IMAGE_WIDTH = 638;
+ private static final int IMAGE_HEIGHT = 1000;
+
+ private static final Correspondence VALIDATE_LANDMARRKS =
+ Correspondence.from(
+ (Correspondence.BinaryPredicate)
+ (actual, expected) -> {
+ return Correspondence.tolerance(FACE_LANDMARKS_ERROR_TOLERANCE)
+ .compare(actual.x(), expected.x())
+ && Correspondence.tolerance(FACE_LANDMARKS_ERROR_TOLERANCE)
+ .compare(actual.y(), expected.y());
+ },
+ "landmarks approximately equal to");
+
+ private static final Correspondence VALIDATE_BLENDSHAPES =
+ Correspondence.from(
+ (Correspondence.BinaryPredicate)
+ (actual, expected) ->
+ Correspondence.tolerance(FACE_BLENDSHAPES_ERROR_TOLERANCE)
+ .compare(actual.score(), expected.score())
+ && actual.index() == expected.index()
+ && actual.categoryName().equals(expected.categoryName()),
+ "face blendshapes approximately equal to");
+
+ @RunWith(AndroidJUnit4.class)
+ public static final class General extends HolisticLandmarkerTest {
+
+ @Test
+ public void detect_successWithValidModels() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE));
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false);
+ assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+ }
+
+ @Test
+ public void detect_successWithBlendshapes() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setOutputFaceBlendshapes(true)
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE));
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ true, /* hasSegmentationMask= */ false);
+ assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+ }
+
+ @Test
+ public void detect_successWithSegmentationMasks() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setOutputPoseSegmentationMasks(true)
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE));
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ true);
+ assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+ }
+
+ @Test
+ public void detect_successWithEmptyResult() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detect(getImageFromAsset(CAT_IMAGE));
+ assertThat(actualResult.faceLandmarks()).isEmpty();
+ }
+
+ @Test
+ public void detect_failsWithRegionOfInterest() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ ImageProcessingOptions imageProcessingOptions =
+ ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build();
+ IllegalArgumentException exception =
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE), imageProcessingOptions));
+ assertThat(exception)
+ .hasMessageThat()
+ .contains("HolisticLandmarker doesn't support region-of-interest");
+ }
+ }
+
+ @RunWith(AndroidJUnit4.class)
+ public static final class RunningModeTest extends HolisticLandmarkerTest {
+ private void assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode runningMode)
+ throws Exception {
+ IllegalArgumentException exception =
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(runningMode)
+ .setResultListener((HolisticLandmarkerResult, inputImage) -> {})
+ .build());
+ assertThat(exception)
+ .hasMessageThat()
+ .contains("a user-defined result listener shouldn't be provided");
+ }
+
+ @Test
+ public void create_failsWithIllegalResultListenerInVideoMode() throws Exception {
+ assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode.VIDEO);
+ }
+
+ @Test
+ public void create_failsWithIllegalResultListenerInImageMode() throws Exception {
+ assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode.IMAGE);
+ }
+
+ @Test
+ public void create_failsWithMissingResultListenerInLiveSteamMode() throws Exception {
+ IllegalArgumentException exception =
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.LIVE_STREAM)
+ .build());
+ assertThat(exception)
+ .hasMessageThat()
+ .contains("a user-defined result listener must be provided");
+ }
+
+ @Test
+ public void detect_failsWithCallingWrongApiInImageMode() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.IMAGE)
+ .build();
+
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ MediaPipeException exception =
+ assertThrows(
+ MediaPipeException.class,
+ () ->
+ holisticLandmarker.detectForVideo(
+ getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0));
+ assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
+ exception =
+ assertThrows(
+ MediaPipeException.class,
+ () ->
+ holisticLandmarker.detectAsync(
+ getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0));
+ assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
+ }
+
+ @Test
+ public void detect_failsWithCallingWrongApiInVideoMode() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.VIDEO)
+ .build();
+
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ MediaPipeException exception =
+ assertThrows(
+ MediaPipeException.class,
+ () -> holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)));
+ assertThat(exception).hasMessageThat().contains("not initialized with the image mode");
+ exception =
+ assertThrows(
+ MediaPipeException.class,
+ () ->
+ holisticLandmarker.detectAsync(
+ getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0));
+ assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode");
+ }
+
+ @Test
+ public void detect_failsWithCallingWrongApiInLiveSteamMode() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.LIVE_STREAM)
+ .setResultListener((HolisticLandmarkerResult, inputImage) -> {})
+ .build();
+
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ MediaPipeException exception =
+ assertThrows(
+ MediaPipeException.class,
+ () -> holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)));
+ assertThat(exception).hasMessageThat().contains("not initialized with the image mode");
+ exception =
+ assertThrows(
+ MediaPipeException.class,
+ () ->
+ holisticLandmarker.detectForVideo(
+ getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0));
+ assertThat(exception).hasMessageThat().contains("not initialized with the video mode");
+ }
+
+ @Test
+ public void detect_successWithImageMode() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.IMAGE)
+ .build();
+
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE));
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false);
+ assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+ }
+
+ @Test
+ public void detect_successWithVideoMode() throws Exception {
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.VIDEO)
+ .build();
+ HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options);
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false);
+ for (int i = 0; i < 3; i++) {
+ HolisticLandmarkerResult actualResult =
+ holisticLandmarker.detectForVideo(getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ i);
+ assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult);
+ }
+ }
+
+ @Test
+ public void detect_failsWithOutOfOrderInputTimestamps() throws Exception {
+ MPImage image = getImageFromAsset(POSE_IMAGE);
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.LIVE_STREAM)
+ .setResultListener((actualResult, inputImage) -> {})
+ .build();
+ try (HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options)) {
+ holisticLandmarker.detectAsync(image, /* timestampsMs= */ 1);
+ MediaPipeException exception =
+ assertThrows(
+ MediaPipeException.class,
+ () -> holisticLandmarker.detectAsync(image, /* timestampsMs= */ 0));
+ assertThat(exception)
+ .hasMessageThat()
+ .contains("having a smaller timestamp than the processed timestamp");
+ }
+ }
+
+ @Test
+ public void detect_successWithLiveSteamMode() throws Exception {
+ MPImage image = getImageFromAsset(POSE_IMAGE);
+ HolisticLandmarkerResult expectedResult =
+ getExpectedHolisticLandmarkerResult(
+ HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false);
+ HolisticLandmarkerOptions options =
+ HolisticLandmarkerOptions.builder()
+ .setBaseOptions(
+ BaseOptions.builder()
+ .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE)
+ .build())
+ .setRunningMode(RunningMode.LIVE_STREAM)
+ .setResultListener(
+ (actualResult, inputImage) -> {
+ assertActualResultApproximatelyEqualsToExpectedResult(
+ actualResult, expectedResult);
+ assertImageSizeIsExpected(inputImage);
+ })
+ .build();
+ try (HolisticLandmarker holisticLandmarker =
+ HolisticLandmarker.createFromOptions(
+ ApplicationProvider.getApplicationContext(), options)) {
+ for (int i = 0; i < 3; i++) {
+ holisticLandmarker.detectAsync(image, /* timestampsMs= */ i);
+ }
+ }
+ }
+ }
+
+ private static MPImage getImageFromAsset(String filePath) throws Exception {
+ AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
+ InputStream istr = assetManager.open(filePath);
+ return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build();
+ }
+
+ private static HolisticLandmarkerResult getExpectedHolisticLandmarkerResult(
+ String resultPath, boolean hasFaceBlendshapes, boolean hasSegmentationMask) throws Exception {
+ AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets();
+
+ HolisticResult holisticResult = HolisticResult.parseFrom(assetManager.open(resultPath));
+
+ Optional blendshapes =
+ hasFaceBlendshapes
+ ? Optional.of(holisticResult.getFaceBlendshapes())
+ : Optional.empty();
+ Optional segmentationMask =
+ hasSegmentationMask ? Optional.of(PLACEHOLDER_MASK) : Optional.empty();
+
+ return HolisticLandmarkerResult.create(
+ holisticResult.getFaceLandmarks(),
+ blendshapes,
+ holisticResult.getPoseLandmarks(),
+ LandmarkList.getDefaultInstance(),
+ segmentationMask,
+ holisticResult.getLeftHandLandmarks(),
+ LandmarkList.getDefaultInstance(),
+ holisticResult.getRightHandLandmarks(),
+ LandmarkList.getDefaultInstance(),
+ /* timestampMs= */ 0);
+ }
+
+ private static void assertActualResultApproximatelyEqualsToExpectedResult(
+ HolisticLandmarkerResult actualResult, HolisticLandmarkerResult expectedResult) {
+ // Expects to have the same number of holistics detected.
+ assertThat(actualResult.faceLandmarks()).hasSize(expectedResult.faceLandmarks().size());
+ assertThat(actualResult.faceBlendshapes().isPresent())
+ .isEqualTo(expectedResult.faceBlendshapes().isPresent());
+ assertThat(actualResult.poseLandmarks()).hasSize(expectedResult.poseLandmarks().size());
+ assertThat(actualResult.segmentationMask().isPresent())
+ .isEqualTo(expectedResult.segmentationMask().isPresent());
+ assertThat(actualResult.leftHandLandmarks()).hasSize(expectedResult.leftHandLandmarks().size());
+ assertThat(actualResult.rightHandLandmarks())
+ .hasSize(expectedResult.rightHandLandmarks().size());
+
+ // Actual face landmarks match expected face landmarks.
+ assertThat(actualResult.faceLandmarks())
+ .comparingElementsUsing(VALIDATE_LANDMARRKS)
+ .containsExactlyElementsIn(expectedResult.faceLandmarks());
+
+ // Actual face blendshapes match expected face blendshapes.
+ if (actualResult.faceBlendshapes().isPresent()) {
+ assertThat(actualResult.faceBlendshapes().get())
+ .comparingElementsUsing(VALIDATE_BLENDSHAPES)
+ .containsExactlyElementsIn(expectedResult.faceBlendshapes().get());
+ }
+
+ // Actual pose landmarks match expected pose landmarks.
+ assertThat(actualResult.poseLandmarks())
+ .comparingElementsUsing(VALIDATE_LANDMARRKS)
+ .containsExactlyElementsIn(expectedResult.poseLandmarks());
+
+ if (actualResult.segmentationMask().isPresent()) {
+ assertImageSizeIsExpected(actualResult.segmentationMask().get());
+ }
+
+ // Actual left hand landmarks match expected left hand landmarks.
+ assertThat(actualResult.leftHandLandmarks())
+ .comparingElementsUsing(VALIDATE_LANDMARRKS)
+ .containsExactlyElementsIn(expectedResult.leftHandLandmarks());
+
+ // Actual right hand landmarks match expected right hand landmarks.
+ assertThat(actualResult.rightHandLandmarks())
+ .comparingElementsUsing(VALIDATE_LANDMARRKS)
+ .containsExactlyElementsIn(expectedResult.rightHandLandmarks());
+ }
+
+ private static void assertImageSizeIsExpected(MPImage inputImage) {
+ assertThat(inputImage).isNotNull();
+ assertThat(inputImage.getWidth()).isEqualTo(IMAGE_WIDTH);
+ assertThat(inputImage.getHeight()).isEqualTo(IMAGE_HEIGHT);
+ }
+}
diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD
index 3f83118b0..422241081 100644
--- a/mediapipe/tasks/testdata/vision/BUILD
+++ b/mediapipe/tasks/testdata/vision/BUILD
@@ -224,6 +224,7 @@ filegroup(
"hand_detector_result_one_hand.pbtxt",
"hand_detector_result_one_hand_rotated.pbtxt",
"hand_detector_result_two_hands.pbtxt",
+ "male_full_height_hands_result_cpu.pbtxt",
"pointing_up_landmarks.pbtxt",
"pointing_up_rotated_landmarks.pbtxt",
"portrait_expected_detection.pbtxt",