+#include "Eigen/Core"
#include "Eigen/Dense"
-#include "Eigen/src/Core/util/Constants.h"
-#include "Eigen/src/Geometry/Quaternion.h"
+#include "Eigen/Geometry"
#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
diff --git a/mediapipe/graphs/selfie_segmentation/BUILD b/mediapipe/graphs/selfie_segmentation/BUILD
new file mode 100644
index 000000000..ddca178de
--- /dev/null
+++ b/mediapipe/graphs/selfie_segmentation/BUILD
@@ -0,0 +1,54 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+ "//mediapipe/framework/tool:mediapipe_graph.bzl",
+ "mediapipe_binary_graph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+ name = "selfie_segmentation_gpu_deps",
+ deps = [
+ "//mediapipe/calculators/core:flow_limiter_calculator",
+ "//mediapipe/calculators/image:recolor_calculator",
+ "//mediapipe/modules/selfie_segmentation:selfie_segmentation_gpu",
+ ],
+)
+
+mediapipe_binary_graph(
+ name = "selfie_segmentation_gpu_binary_graph",
+ graph = "selfie_segmentation_gpu.pbtxt",
+ output_name = "selfie_segmentation_gpu.binarypb",
+ deps = [":selfie_segmentation_gpu_deps"],
+)
+
+cc_library(
+ name = "selfie_segmentation_cpu_deps",
+ deps = [
+ "//mediapipe/calculators/core:flow_limiter_calculator",
+ "//mediapipe/calculators/image:recolor_calculator",
+ "//mediapipe/modules/selfie_segmentation:selfie_segmentation_cpu",
+ ],
+)
+
+mediapipe_binary_graph(
+ name = "selfie_segmentation_cpu_binary_graph",
+ graph = "selfie_segmentation_cpu.pbtxt",
+ output_name = "selfie_segmentation_cpu.binarypb",
+ deps = [":selfie_segmentation_cpu_deps"],
+)
diff --git a/mediapipe/graphs/selfie_segmentation/selfie_segmentation_cpu.pbtxt b/mediapipe/graphs/selfie_segmentation/selfie_segmentation_cpu.pbtxt
new file mode 100644
index 000000000..db1b479a1
--- /dev/null
+++ b/mediapipe/graphs/selfie_segmentation/selfie_segmentation_cpu.pbtxt
@@ -0,0 +1,52 @@
+# MediaPipe graph that performs selfie segmentation with TensorFlow Lite on CPU.
+
+# CPU buffer. (ImageFrame)
+input_stream: "input_video"
+
+# Output image with rendered results. (ImageFrame)
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+ calculator: "FlowLimiterCalculator"
+ input_stream: "input_video"
+ input_stream: "FINISHED:output_video"
+ input_stream_info: {
+ tag_index: "FINISHED"
+ back_edge: true
+ }
+ output_stream: "throttled_input_video"
+}
+
+# Subgraph that performs selfie segmentation.
+node {
+ calculator: "SelfieSegmentationCpu"
+ input_stream: "IMAGE:throttled_input_video"
+ output_stream: "SEGMENTATION_MASK:segmentation_mask"
+}
+
+
+# Colors the selfie segmentation with the color specified in the option.
+node {
+ calculator: "RecolorCalculator"
+ input_stream: "IMAGE:throttled_input_video"
+ input_stream: "MASK:segmentation_mask"
+ output_stream: "IMAGE:output_video"
+ node_options: {
+ [type.googleapis.com/mediapipe.RecolorCalculatorOptions] {
+ color { r: 0 g: 0 b: 255 }
+ mask_channel: RED
+ invert_mask: true
+ adjust_with_luminance: false
+ }
+ }
+}
diff --git a/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt b/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt
new file mode 100644
index 000000000..08d4c36a8
--- /dev/null
+++ b/mediapipe/graphs/selfie_segmentation/selfie_segmentation_gpu.pbtxt
@@ -0,0 +1,52 @@
+# MediaPipe graph that performs selfie segmentation with TensorFlow Lite on GPU.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Output image with rendered results. (GpuBuffer)
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+ calculator: "FlowLimiterCalculator"
+ input_stream: "input_video"
+ input_stream: "FINISHED:output_video"
+ input_stream_info: {
+ tag_index: "FINISHED"
+ back_edge: true
+ }
+ output_stream: "throttled_input_video"
+}
+
+# Subgraph that performs selfie segmentation.
+node {
+ calculator: "SelfieSegmentationGpu"
+ input_stream: "IMAGE:throttled_input_video"
+ output_stream: "SEGMENTATION_MASK:segmentation_mask"
+}
+
+
+# Colors the selfie segmentation with the color specified in the option.
+node {
+ calculator: "RecolorCalculator"
+ input_stream: "IMAGE_GPU:throttled_input_video"
+ input_stream: "MASK_GPU:segmentation_mask"
+ output_stream: "IMAGE_GPU:output_video"
+ node_options: {
+ [type.googleapis.com/mediapipe.RecolorCalculatorOptions] {
+ color { r: 0 g: 0 b: 255 }
+ mask_channel: RED
+ invert_mask: true
+ adjust_with_luminance: false
+ }
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
new file mode 100644
index 000000000..694ffc5d9
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
@@ -0,0 +1,223 @@
+// Copyright 2019-2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.components;
+
+import android.graphics.SurfaceTexture;
+import android.opengl.GLES11Ext;
+import android.opengl.GLES20;
+import android.opengl.GLSurfaceView;
+import android.opengl.Matrix;
+import android.util.Log;
+import com.google.mediapipe.framework.TextureFrame;
+import com.google.mediapipe.glutil.CommonShaders;
+import com.google.mediapipe.glutil.ShaderUtil;
+import java.nio.FloatBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.microedition.khronos.egl.EGLConfig;
+import javax.microedition.khronos.opengles.GL10;
+
+/**
+ * Renderer for a {@link GLSurfaceView}. It displays a texture. The texture is scaled and cropped as
+ * necessary to fill the view, while maintaining its aspect ratio.
+ *
+ * It can render both textures bindable to the normal {@link GLES20#GL_TEXTURE_2D} target as well
+ * as textures bindable to {@link GLES11Ext#GL_TEXTURE_EXTERNAL_OES}, which is used for Android
+ * surfaces. Call {@link #setTextureTarget(int)} to choose the correct target.
+ *
+ *
It can display a {@link SurfaceTexture} (call {@link #setSurfaceTexture(SurfaceTexture)}) or a
+ * {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}).
+ */
+public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
+ private static final String TAG = "DemoRenderer";
+ private static final int ATTRIB_POSITION = 1;
+ private static final int ATTRIB_TEXTURE_COORDINATE = 2;
+
+ private int surfaceWidth;
+ private int surfaceHeight;
+ private int frameWidth = 0;
+ private int frameHeight = 0;
+ private int program = 0;
+ private int frameUniform;
+ private int textureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
+ private int textureTransformUniform;
+ // Controls the alignment between frame size and surface size, 0.5f default is centered.
+ private float alignmentHorizontal = 0.5f;
+ private float alignmentVertical = 0.5f;
+ private float[] textureTransformMatrix = new float[16];
+ private SurfaceTexture surfaceTexture = null;
+ private final AtomicReference nextFrame = new AtomicReference<>();
+
+ @Override
+ public void onSurfaceCreated(GL10 gl, EGLConfig config) {
+ if (surfaceTexture == null) {
+ Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */);
+ }
+ Map attributeLocations = new HashMap<>();
+ attributeLocations.put("position", ATTRIB_POSITION);
+ attributeLocations.put("texture_coordinate", ATTRIB_TEXTURE_COORDINATE);
+ Log.d(TAG, "external texture: " + isExternalTexture());
+ program =
+ ShaderUtil.createProgram(
+ CommonShaders.VERTEX_SHADER,
+ isExternalTexture()
+ ? CommonShaders.FRAGMENT_SHADER_EXTERNAL
+ : CommonShaders.FRAGMENT_SHADER,
+ attributeLocations);
+ frameUniform = GLES20.glGetUniformLocation(program, "video_frame");
+ textureTransformUniform = GLES20.glGetUniformLocation(program, "texture_transform");
+ ShaderUtil.checkGlError("glGetUniformLocation");
+
+ GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
+ }
+
+ @Override
+ public void onSurfaceChanged(GL10 gl, int width, int height) {
+ surfaceWidth = width;
+ surfaceHeight = height;
+ GLES20.glViewport(0, 0, width, height);
+ }
+
+ @Override
+ public void onDrawFrame(GL10 gl) {
+ TextureFrame frame = nextFrame.getAndSet(null);
+
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+ ShaderUtil.checkGlError("glClear");
+
+ if (surfaceTexture == null && frame == null) {
+ return;
+ }
+
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
+ ShaderUtil.checkGlError("glActiveTexture");
+ if (surfaceTexture != null) {
+ surfaceTexture.updateTexImage();
+ surfaceTexture.getTransformMatrix(textureTransformMatrix);
+ } else {
+ GLES20.glBindTexture(textureTarget, frame.getTextureName());
+ ShaderUtil.checkGlError("glBindTexture");
+ }
+ GLES20.glTexParameteri(textureTarget, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
+ GLES20.glTexParameteri(textureTarget, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
+ GLES20.glTexParameteri(textureTarget, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
+ GLES20.glTexParameteri(textureTarget, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
+ ShaderUtil.checkGlError("texture setup");
+
+ GLES20.glUseProgram(program);
+ GLES20.glUniform1i(frameUniform, 0);
+ GLES20.glUniformMatrix4fv(textureTransformUniform, 1, false, textureTransformMatrix, 0);
+ ShaderUtil.checkGlError("glUniformMatrix4fv");
+ GLES20.glEnableVertexAttribArray(ATTRIB_POSITION);
+ GLES20.glVertexAttribPointer(
+ ATTRIB_POSITION, 2, GLES20.GL_FLOAT, false, 0, CommonShaders.SQUARE_VERTICES);
+
+ // TODO: compute scale from surfaceTexture size.
+ float scaleWidth = frameWidth > 0 ? (float) surfaceWidth / (float) frameWidth : 1.0f;
+ float scaleHeight = frameHeight > 0 ? (float) surfaceHeight / (float) frameHeight : 1.0f;
+ // Whichever of the two scales is greater corresponds to the dimension where the image
+ // is proportionally smaller than the view. Dividing both scales by that number results
+ // in that dimension having scale 1.0, and thus touching the edges of the view, while the
+ // other is cropped proportionally.
+ float maxScale = Math.max(scaleWidth, scaleHeight);
+ scaleWidth /= maxScale;
+ scaleHeight /= maxScale;
+
+ // Alignment controls where the visible section is placed within the full camera frame, with
+ // (0, 0) being the bottom left, and (1, 1) being the top right.
+ float textureLeft = (1.0f - scaleWidth) * alignmentHorizontal;
+ float textureRight = textureLeft + scaleWidth;
+ float textureBottom = (1.0f - scaleHeight) * alignmentVertical;
+ float textureTop = textureBottom + scaleHeight;
+
+ // Unlike on iOS, there is no need to flip the surfaceTexture here.
+ // But for regular textures, we will need to flip them.
+ final FloatBuffer passThroughTextureVertices =
+ ShaderUtil.floatBuffer(
+ textureLeft, textureBottom,
+ textureRight, textureBottom,
+ textureLeft, textureTop,
+ textureRight, textureTop);
+ GLES20.glEnableVertexAttribArray(ATTRIB_TEXTURE_COORDINATE);
+ GLES20.glVertexAttribPointer(
+ ATTRIB_TEXTURE_COORDINATE, 2, GLES20.GL_FLOAT, false, 0, passThroughTextureVertices);
+ ShaderUtil.checkGlError("program setup");
+
+ GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
+ ShaderUtil.checkGlError("glDrawArrays");
+ GLES20.glBindTexture(textureTarget, 0);
+ ShaderUtil.checkGlError("unbind surfaceTexture");
+
+ // We must flush before releasing the frame.
+ GLES20.glFlush();
+
+ if (frame != null) {
+ frame.release();
+ }
+ }
+
+ public void setTextureTarget(int target) {
+ if (program != 0) {
+ throw new IllegalStateException(
+ "setTextureTarget must be called before the surface is created");
+ }
+ textureTarget = target;
+ }
+
+ public void setSurfaceTexture(SurfaceTexture texture) {
+ if (!isExternalTexture()) {
+ throw new IllegalStateException(
+ "to use a SurfaceTexture, the texture target must be GL_TEXTURE_EXTERNAL_OES");
+ }
+ TextureFrame oldFrame = nextFrame.getAndSet(null);
+ if (oldFrame != null) {
+ oldFrame.release();
+ }
+ surfaceTexture = texture;
+ }
+
+ // Use this when the texture is not a SurfaceTexture.
+ public void setNextFrame(TextureFrame frame) {
+ if (surfaceTexture != null) {
+ Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */);
+ }
+ TextureFrame oldFrame = nextFrame.getAndSet(frame);
+ if (oldFrame != null
+ && (frame == null || (oldFrame.getTextureName() != frame.getTextureName()))) {
+ oldFrame.release();
+ }
+ surfaceTexture = null;
+ }
+
+ public void setFrameSize(int width, int height) {
+ frameWidth = width;
+ frameHeight = height;
+ }
+
+ /**
+ * When the aspect ratios between the camera frame and the surface size are mismatched, this
+ * controls how the image is aligned. 0.0 means aligning the left/bottom edges; 1.0 means aligning
+ * the right/top edges; 0.5 (default) means aligning the centers.
+ */
+ public void setAlignment(float horizontal, float vertical) {
+ alignmentHorizontal = horizontal;
+ alignmentVertical = vertical;
+ }
+
+ private boolean isExternalTexture() {
+ return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java b/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java
index 69c0ebeb6..5ddeb98c9 100644
--- a/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java
+++ b/mediapipe/java/com/google/mediapipe/framework/AndroidPacketCreator.java
@@ -16,6 +16,7 @@ package com.google.mediapipe.framework;
import android.graphics.Bitmap;
import java.nio.ByteBuffer;
+import java.util.List;
// TODO: use Preconditions in this file.
/**
diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc
index 30ec19a25..1a7fd18b0 100644
--- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc
+++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc
@@ -444,8 +444,16 @@ JNIEXPORT jlong JNICALL PACKET_GETTER_METHOD(nativeGetGpuBuffer)(JNIEnv* env,
mediapipe::android::Graph::GetPacketFromHandle(packet);
mediapipe::GlTextureBufferSharedPtr ptr;
if (mediapipe_packet.ValidateAsType().ok()) {
- const mediapipe::Image& buffer = mediapipe_packet.Get();
- ptr = buffer.GetGlTextureBufferSharedPtr();
+ auto mediapipe_graph =
+ mediapipe::android::Graph::GetContextFromHandle(packet);
+ auto gl_context = mediapipe_graph->GetGpuResources()->gl_context();
+ auto status =
+ gl_context->Run([gl_context, mediapipe_packet, &ptr]() -> absl::Status {
+ const mediapipe::Image& buffer =
+ mediapipe_packet.Get();
+ ptr = buffer.GetGlTextureBufferSharedPtr();
+ return absl::OkStatus();
+ });
} else {
const mediapipe::GpuBuffer& buffer =
mediapipe_packet.Get();
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/BUILD b/mediapipe/java/com/google/mediapipe/solutionbase/BUILD
new file mode 100644
index 000000000..a3acad5d0
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/BUILD
@@ -0,0 +1,67 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+android_library(
+ name = "solution_base",
+ srcs = glob(
+ ["*.java"],
+ exclude = [
+ "CameraInput.java",
+ ],
+ ),
+ visibility = ["//visibility:public"],
+ deps = [
+ "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+ "//mediapipe/java/com/google/mediapipe/glutil",
+ "//third_party:autovalue",
+ "@maven//:com_google_code_findbugs_jsr305",
+ "@maven//:com_google_guava_guava",
+ ],
+)
+
+android_library(
+ name = "camera_input",
+ srcs = ["CameraInput.java"],
+ visibility = ["//visibility:public"],
+ deps = [
+ "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+ "//mediapipe/java/com/google/mediapipe/components:android_components",
+ "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+ "@maven//:com_google_guava_guava",
+ ],
+)
+
+# Native dependencies of all MediaPipe solutions.
+cc_binary(
+ name = "libmediapipe_jni.so",
+ linkshared = 1,
+ linkstatic = 1,
+ # TODO: Add more calculators to support other top-level solutions.
+ deps = [
+ "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+ "//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image",
+ ],
+)
+
+# Converts the .so cc_binary into a cc_library, to be consumed in an android_binary.
+cc_library(
+ name = "mediapipe_jni_lib",
+ srcs = [":libmediapipe_jni.so"],
+ visibility = ["//visibility:public"],
+ alwayslink = 1,
+)
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/CameraInput.java b/mediapipe/java/com/google/mediapipe/solutionbase/CameraInput.java
new file mode 100644
index 000000000..acfbde74d
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/CameraInput.java
@@ -0,0 +1,109 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import android.app.Activity;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.components.TextureFrameConsumer;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.TextureFrame;
+import javax.microedition.khronos.egl.EGLContext;
+
+/**
+ * The camera component that takes the camera input and produces MediaPipe {@link TextureFrame}
+ * objects.
+ */
+public class CameraInput {
+ private static final String TAG = "CameraInput";
+
+ /** Represents the direction the camera faces relative to device screen. */
+ public static enum CameraFacing {
+ FRONT,
+ BACK
+ };
+
+ private final CameraXPreviewHelper cameraHelper;
+ private TextureFrameConsumer cameraNewFrameListener;
+ private ExternalTextureConverter converter;
+
+ /**
+ * Initializes CamereInput and requests camera permissions.
+ *
+ * @param activity an Android {@link Activity}.
+ */
+ public CameraInput(Activity activity) {
+ cameraHelper = new CameraXPreviewHelper();
+ PermissionHelper.checkAndRequestCameraPermissions(activity);
+ }
+
+ /**
+ * Sets a callback to be invoked when new frames available.
+ *
+ * @param listener the callback.
+ */
+ public void setCameraNewFrameListener(TextureFrameConsumer listener) {
+ cameraNewFrameListener = listener;
+ }
+
+ /**
+ * Sets up the external texture converter and starts the camera.
+ *
+ * @param activity an Android {@link Activity}.
+ * @param eglContext an OpenGL {@link EGLContext}.
+ * @param cameraFacing the direction the camera faces relative to device screen.
+ * @param width the desired width of the converted texture.
+ * @param height the desired height of the converted texture.
+ */
+ public void start(
+ Activity activity, EGLContext eglContext, CameraFacing cameraFacing, int width, int height) {
+ if (!PermissionHelper.cameraPermissionsGranted(activity)) {
+ return;
+ }
+ if (converter == null) {
+ converter = new ExternalTextureConverter(eglContext, 2);
+ }
+ if (cameraNewFrameListener == null) {
+ throw new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "cameraNewFrameListener is not set.");
+ }
+ converter.setConsumer(cameraNewFrameListener);
+ cameraHelper.setOnCameraStartedListener(
+ surfaceTexture ->
+ converter.setSurfaceTextureAndAttachToGLContext(surfaceTexture, width, height));
+ cameraHelper.startCamera(
+ activity,
+ cameraFacing == CameraFacing.FRONT
+ ? CameraHelper.CameraFacing.FRONT
+ : CameraHelper.CameraFacing.BACK,
+ /*unusedSurfaceTexture=*/ null,
+ null);
+ }
+
+ /** Stops the camera input. */
+ public void stop() {
+ if (converter != null) {
+ converter.close();
+ }
+ }
+
+ /** Returns a boolean which is true if the camera is in Portrait mode, false in Landscape mode. */
+ public boolean isCameraRotated() {
+ return cameraHelper.isCameraRotated();
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ErrorListener.java b/mediapipe/java/com/google/mediapipe/solutionbase/ErrorListener.java
new file mode 100644
index 000000000..62723cdfb
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/ErrorListener.java
@@ -0,0 +1,20 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+/** Interface for the customizable MediaPipe solution error listener. */
+public interface ErrorListener {
+ void onError(String message, RuntimeException e);
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionBase.java b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionBase.java
new file mode 100644
index 000000000..11d1808bf
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionBase.java
@@ -0,0 +1,174 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.util.Log;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.TextureFrame;
+import com.google.mediapipe.glutil.EglManager;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.microedition.khronos.egl.EGLContext;
+
+/** The base class of the MediaPipe image solutions. */
+// TODO: Consolidates the "send" methods to be a single "send(MlImage image)".
+public class ImageSolutionBase extends SolutionBase {
+ public static final String TAG = "ImageSolutionBase";
+ protected boolean staticImageMode;
+ private EglManager eglManager;
+ // Internal fake timestamp for static images.
+ private final AtomicInteger staticImageTimestamp = new AtomicInteger(0);
+
+ /**
+ * Initializes MediaPipe image solution base with Android context, solution specific settings, and
+ * solution result handler.
+ *
+ * @param context an Android {@link Context}.
+ * @param solutionInfo a {@link SolutionInfo} contains binary graph file path, graph input and
+ * output stream names.
+ * @param outputHandler a {@link OutputHandler} handles the solution graph output packets and
+ * runtime exception.
+ */
+ @Override
+ public synchronized void initialize(
+ Context context,
+ SolutionInfo solutionInfo,
+ OutputHandler outputHandler) {
+ staticImageMode = solutionInfo.staticImageMode();
+ try {
+ super.initialize(context, solutionInfo, outputHandler);
+ eglManager = new EglManager(/*parentContext=*/ null);
+ solutionGraph.setParentGlContext(eglManager.getNativeContext());
+ } catch (MediaPipeException e) {
+ throwException("Error occurs when creating MediaPipe image solution graph. ", e);
+ }
+ }
+
+ /** Returns the managed {@link EGLContext} to share the opengl context with other components. */
+ public EGLContext getGlContext() {
+ return eglManager.getContext();
+ }
+
+
+ /** Returns the opengl major version number. */
+ public int getGlMajorVersion() {
+ return eglManager.getGlMajorVersion();
+ }
+
+ /** Sends a {@link TextureFrame} into solution graph for processing. */
+ public void send(TextureFrame textureFrame) {
+ if (!staticImageMode && textureFrame.getTimestamp() == Long.MIN_VALUE) {
+ throwException(
+ "Error occurs when calling the solution send method. ",
+ new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "TextureFrame's timestamp needs to be explicitly set if not in static image mode."));
+ return;
+ }
+ long timestampUs =
+ staticImageMode ? staticImageTimestamp.getAndIncrement() : textureFrame.getTimestamp();
+ sendImage(textureFrame, timestampUs);
+ }
+
+ /**
+ * Sends a {@link Bitmap} with a timestamp into solution graph for processing. In static image
+ * mode, the timestamp is ignored.
+ */
+ public void send(Bitmap inputBitmap, long timestamp) {
+ if (staticImageMode) {
+ Log.w(TAG, "In static image mode, the MediaPipe solution ignores the input timestamp.");
+ }
+ sendImage(inputBitmap, staticImageMode ? staticImageTimestamp.getAndIncrement() : timestamp);
+ }
+
+ /** Sends a {@link Bitmap} (static image) into solution graph for processing. */
+ public void send(Bitmap inputBitmap) {
+ if (!staticImageMode) {
+ throwException(
+ "Error occurs when calling the solution send method. ",
+ new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "When not in static image mode, a timestamp associated with the image is required."
+ + " Use send(Bitmap inputBitmap, long timestamp) instead."));
+ return;
+ }
+ sendImage(inputBitmap, staticImageTimestamp.getAndIncrement());
+ }
+
+ /** Internal implementation of sending Bitmap/TextureFrame into the MediaPipe solution. */
+ private synchronized void sendImage(T imageObj, long timestamp) {
+ if (lastTimestamp >= timestamp) {
+ throwException(
+ "The received frame having a smaller timestamp than the processed timestamp.",
+ new MediaPipeException(
+ MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(),
+ "Receving a frame with invalid timestamp."));
+ return;
+ }
+ lastTimestamp = timestamp;
+ Packet imagePacket = null;
+ try {
+ if (imageObj instanceof TextureFrame) {
+ imagePacket = packetCreator.createImage((TextureFrame) imageObj);
+ imageObj = null;
+ } else if (imageObj instanceof Bitmap) {
+ imagePacket = packetCreator.createRgbaImage((Bitmap) imageObj);
+ } else {
+ throwException(
+ "The input image type is not supported. ",
+ new MediaPipeException(
+ MediaPipeException.StatusCode.UNIMPLEMENTED.ordinal(),
+ "The input image type is not supported."));
+ }
+
+ try {
+ // addConsumablePacketToInputStream allows the graph to take exclusive ownership of the
+ // packet, which may allow for more memory optimizations.
+ solutionGraph.addConsumablePacketToInputStream(
+ imageInputStreamName, imagePacket, timestamp);
+ // If addConsumablePacket succeeded, we don't need to release the packet ourselves.
+ imagePacket = null;
+ } catch (MediaPipeException e) {
+ // TODO: do not suppress exceptions here!
+ if (errorListener == null) {
+ Log.e(TAG, "Mediapipe error: ", e);
+ } else {
+ throw e;
+ }
+ }
+ } catch (RuntimeException e) {
+ if (errorListener != null) {
+ errorListener.onError("Mediapipe error: ", e);
+ } else {
+ throw e;
+ }
+ } finally {
+ if (imagePacket != null) {
+ // In case of error, addConsumablePacketToInputStream will not release the packet, so we
+ // have to release it ourselves. (We could also re-try adding, but we don't).
+ imagePacket.release();
+ }
+ if (imageObj instanceof TextureFrame) {
+ if (imageObj != null) {
+ // imagePacket will release frame if it has been created, but if not, we need to
+ // release it.
+ ((TextureFrame) imageObj).release();
+ }
+ }
+ }
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java
new file mode 100644
index 000000000..9e8cc11a1
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java
@@ -0,0 +1,59 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import android.graphics.Bitmap;
+import com.google.mediapipe.framework.AndroidPacketGetter;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.framework.TextureFrame;
+
+/**
+ * The base class of any MediaPipe image solution result. The base class contains the common parts
+ * across all image solution results, including the input timestamp and the input image data. A new
+ * MediaPipe image solution result class should extend ImageSolutionResult.
+ */
+public class ImageSolutionResult implements SolutionResult {
+ protected long timestamp;
+ protected Packet imagePacket;
+ private Bitmap cachedBitmap;
+
+ // Result timestamp, which is set to the timestamp of the corresponding input image. May return
+ // Long.MIN_VALUE if the input image is not associated with a timestamp.
+ @Override
+ public long timestamp() {
+ return timestamp;
+ }
+
+ // Returns the corresponding input image as a {@link Bitmap}.
+ public Bitmap inputBitmap() {
+ if (cachedBitmap != null) {
+ return cachedBitmap;
+ }
+ cachedBitmap = AndroidPacketGetter.getBitmapFromRgba(imagePacket);
+ return cachedBitmap;
+ }
+
+ // Returns the corresponding input image as a {@link TextureFrame}. The caller must release the
+ // acquired {@link TextureFrame} after using.
+ public TextureFrame acquireTextureFrame() {
+ return PacketGetter.getTextureFrame(imagePacket);
+ }
+
+ // Releases image packet and the underlying data.
+ void releaseImagePacket() {
+ imagePacket.release();
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/OutputHandler.java b/mediapipe/java/com/google/mediapipe/solutionbase/OutputHandler.java
new file mode 100644
index 000000000..f76d5c7a3
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/OutputHandler.java
@@ -0,0 +1,86 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import android.util.Log;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.Packet;
+import java.util.List;
+
+/** Interface for handling MediaPipe solution graph outputs. */
+public class OutputHandler {
+ private static final String TAG = "OutputHandler";
+
+ /** Interface for converting outputs packet lists to solution result objects. */
+ public interface OutputConverter {
+ public abstract T convert(List packets);
+ }
+ // A solution specific graph output converter that should be implemented by solution.
+ private OutputConverter outputConverter;
+ // The user-defined solution result listener.
+ private ResultListener customResultListener;
+ // The user-defined error listener.
+ private ErrorListener customErrorListener;
+
+ /**
+ * Sets a callback to be invoked to convert a packet list to a solution result object.
+ *
+ * @param converter the solution-defined {@link OutputConverter} callback.
+ */
+ public void setOutputConverter(OutputConverter converter) {
+ this.outputConverter = converter;
+ }
+
+ /**
+ * Sets a callback to be invoked when a solution result objects become available .
+ *
+ * @param listener the user-defined {@link ResultListener} callback.
+ */
+ public void setResultListener(ResultListener listener) {
+ this.customResultListener = listener;
+ }
+
+ /**
+ * Sets a callback to be invoked when exceptions are thrown in the solution.
+ *
+ * @param listener the user-defined {@link ErrorListener} callback.
+ */
+ public void setErrorListener(ErrorListener listener) {
+ this.customErrorListener = listener;
+ }
+
+ /** Handles a list of output packets. Invoked when packet lists become available. */
+ public void run(List packets) {
+ T solutionResult = null;
+ try {
+ solutionResult = outputConverter.convert(packets);
+ customResultListener.run(solutionResult);
+ } catch (MediaPipeException e) {
+ if (customErrorListener != null) {
+ customErrorListener.onError("Error occurs when getting MediaPipe solution result. ", e);
+ } else {
+ Log.e(TAG, "Error occurs when getting MediaPipe solution result. " + e);
+ }
+ } finally {
+ for (Packet packet : packets) {
+ packet.release();
+ }
+ if (solutionResult instanceof ImageSolutionResult) {
+ ImageSolutionResult imageSolutionResult = (ImageSolutionResult) solutionResult;
+ imageSolutionResult.releaseImagePacket();
+ }
+ }
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/ResultListener.java b/mediapipe/java/com/google/mediapipe/solutionbase/ResultListener.java
new file mode 100644
index 000000000..938c115f9
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/ResultListener.java
@@ -0,0 +1,20 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+/** Interface for the customizable MediaPipe solution result listener. */
+public interface ResultListener {
+ void run(T result);
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java
new file mode 100644
index 000000000..d42194567
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java
@@ -0,0 +1,150 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import static java.util.concurrent.TimeUnit.MICROSECONDS;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import android.content.Context;
+import android.os.SystemClock;
+import android.util.Log;
+import com.google.common.collect.ImmutableList;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.framework.AndroidPacketCreator;
+import com.google.mediapipe.framework.Graph;
+import com.google.mediapipe.framework.MediaPipeException;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.protobuf.Parser;
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
+
+/** The base class of the MediaPipe solutions. */
+public class SolutionBase {
+ private static final String TAG = "SolutionBase";
+ protected Graph solutionGraph;
+ protected AndroidPacketCreator packetCreator;
+ protected ErrorListener errorListener;
+ protected String imageInputStreamName;
+ protected long lastTimestamp = Long.MIN_VALUE;
+ protected final AtomicBoolean solutionGraphStarted = new AtomicBoolean(false);
+
+ static {
+ // Load all native libraries needed by the app.
+ System.loadLibrary("mediapipe_jni");
+ System.loadLibrary("opencv_java3");
+ }
+
+ /**
+ * Initializes solution base with Android context, solution specific settings, and solution result
+ * handler.
+ *
+ * @param context an Android {@link Context}.
+ * @param solutionInfo a {@link SolutionInfo} contains binary graph file path, graph input and
+ * output stream names.
+ * @param outputHandler a {@link OutputHandler} handles both solution result object and runtime
+ * exception.
+ */
+ public synchronized void initialize(
+ Context context,
+ SolutionInfo solutionInfo,
+ OutputHandler outputHandler) {
+ this.imageInputStreamName = solutionInfo.imageInputStreamName();
+ try {
+ AndroidAssetUtil.initializeNativeAssetManager(context);
+ solutionGraph = new Graph();
+ if (new File(solutionInfo.binaryGraphPath()).isAbsolute()) {
+ solutionGraph.loadBinaryGraph(solutionInfo.binaryGraphPath());
+ } else {
+ solutionGraph.loadBinaryGraph(
+ AndroidAssetUtil.getAssetBytes(context.getAssets(), solutionInfo.binaryGraphPath()));
+ }
+ solutionGraph.addMultiStreamCallback(
+ solutionInfo.outputStreamNames(), outputHandler::run, /*observeTimestampBounds=*/ true);
+ packetCreator = new AndroidPacketCreator(solutionGraph);
+ } catch (MediaPipeException e) {
+ throwException("Error occurs when creating the MediaPipe solution graph. ", e);
+ }
+ }
+
+ /** Throws exception with error message. */
+ protected void throwException(String message, MediaPipeException e) {
+ if (errorListener != null) {
+ errorListener.onError(message, e);
+ } else {
+ Log.e(TAG, message, e);
+ }
+ }
+
+ /**
+ * A convinence method to get proto list from a packet. If packet is empty, returns an empty list.
+ */
+ protected List getProtoVector(Packet packet, Parser messageParser) {
+ return packet.isEmpty()
+ ? ImmutableList.of()
+ : PacketGetter.getProtoVector(packet, messageParser);
+ }
+
+ /** Gets current timestamp in microseconds. */
+ protected long getCurrentTimestampUs() {
+ return MICROSECONDS.convert(SystemClock.elapsedRealtime(), MILLISECONDS);
+ }
+
+ /** Starts the solution graph by taking an optional input side packets map. */
+ public synchronized void start(@Nullable Map inputSidePackets) {
+ try {
+ if (inputSidePackets != null) {
+ solutionGraph.setInputSidePackets(inputSidePackets);
+ }
+ if (!solutionGraphStarted.getAndSet(true)) {
+ solutionGraph.startRunningGraph();
+ }
+ } catch (MediaPipeException e) {
+ throwException("Error occurs when starting the MediaPipe solution graph. ", e);
+ }
+ }
+
+ /** A blocking API that returns until the solution finishes processing all the pending tasks. */
+ public void waitUntilIdle() {
+ try {
+ solutionGraph.waitUntilGraphIdle();
+ } catch (MediaPipeException e) {
+ throwException("Error occurs when waiting until the MediaPipe graph becomes idle. ", e);
+ }
+ }
+
+ /** Closes and cleans up the solution graph. */
+ public void close() {
+ if (solutionGraphStarted.get()) {
+ try {
+ solutionGraph.closeAllPacketSources();
+ solutionGraph.waitUntilGraphDone();
+ } catch (MediaPipeException e) {
+ // Note: errors during Process are reported at the earliest opportunity,
+ // which may be addPacket or waitUntilDone, depending on timing. For consistency,
+ // we want to always report them using the same async handler if installed.
+ throwException("Error occurs when closing the Mediapipe solution graph. ", e);
+ }
+ try {
+ solutionGraph.tearDown();
+ } catch (MediaPipeException e) {
+ throwException("Error occurs when closing the Mediapipe solution graph. ", e);
+ }
+ }
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionInfo.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionInfo.java
new file mode 100644
index 000000000..fed2994b2
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionInfo.java
@@ -0,0 +1,48 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+
+/** SolutionInfo contains all needed informaton to initialize a MediaPipe solution graph. */
+@AutoValue
+public abstract class SolutionInfo {
+ public abstract String binaryGraphPath();
+
+ public abstract String imageInputStreamName();
+
+ public abstract ImmutableList outputStreamNames();
+
+ public abstract boolean staticImageMode();
+
+ public static Builder builder() {
+ return new AutoValue_SolutionInfo.Builder();
+ }
+
+ /** Builder for {@link SolutionInfo}. */
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder setBinaryGraphPath(String value);
+
+ public abstract Builder setImageInputStreamName(String value);
+
+ public abstract Builder setOutputStreamNames(ImmutableList value);
+
+ public abstract Builder setStaticImageMode(boolean value);
+
+ public abstract SolutionInfo build();
+ }
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutionbase/SolutionResult.java b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionResult.java
new file mode 100644
index 000000000..c77b79847
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutionbase/SolutionResult.java
@@ -0,0 +1,23 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.solutionbase;
+
+/**
+ * Interface of the MediaPipe solution result. Any MediaPipe solution-specific result class should
+ * implement SolutionResult.
+ */
+public interface SolutionResult {
+ long timestamp();
+}
diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD
index 374cfeb58..9ee606455 100644
--- a/mediapipe/modules/face_detection/BUILD
+++ b/mediapipe/modules/face_detection/BUILD
@@ -83,6 +83,8 @@ mediapipe_simple_subgraph(
exports_files(
srcs = [
+ "face_detection_back.tflite",
+ "face_detection_back_sparse.tflite",
"face_detection_front.tflite",
],
)
diff --git a/mediapipe/modules/face_detection/face_detection_back.tflite b/mediapipe/modules/face_detection/face_detection_back.tflite
index b15764ebd..98c5c16bb 100755
Binary files a/mediapipe/modules/face_detection/face_detection_back.tflite and b/mediapipe/modules/face_detection/face_detection_back.tflite differ
diff --git a/mediapipe/modules/face_detection/face_detection_back_sparse.tflite b/mediapipe/modules/face_detection/face_detection_back_sparse.tflite
new file mode 100755
index 000000000..9575d8c1f
Binary files /dev/null and b/mediapipe/modules/face_detection/face_detection_back_sparse.tflite differ
diff --git a/mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt
index 4eb29be65..a94a8c803 100644
--- a/mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt
+++ b/mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt
@@ -109,7 +109,7 @@ node {
output_stream: "ensured_landmark_tensors"
}
-# Decodes the landmark tensors into a vector of lanmarks, where the landmark
+# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
diff --git a/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt
index 17c9bd78c..7d8c3bf7d 100644
--- a/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt
+++ b/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt
@@ -109,7 +109,7 @@ node {
output_stream: "ensured_landmark_tensors"
}
-# Decodes the landmark tensors into a vector of lanmarks, where the landmark
+# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
diff --git a/mediapipe/modules/objectron/calculators/box.cc b/mediapipe/modules/objectron/calculators/box.cc
index a7d0f1460..bd2ce57f9 100644
--- a/mediapipe/modules/objectron/calculators/box.cc
+++ b/mediapipe/modules/objectron/calculators/box.cc
@@ -14,7 +14,7 @@
#include "mediapipe/modules/objectron/calculators/box.h"
-#include "Eigen/src/Core/util/Constants.h"
+#include "Eigen/Core"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
diff --git a/mediapipe/modules/pose_landmark/BUILD b/mediapipe/modules/pose_landmark/BUILD
index 90edbb8a0..f38b2040d 100644
--- a/mediapipe/modules/pose_landmark/BUILD
+++ b/mediapipe/modules/pose_landmark/BUILD
@@ -78,7 +78,9 @@ mediapipe_simple_subgraph(
graph = "pose_landmark_filtering.pbtxt",
register_as = "PoseLandmarkFiltering",
deps = [
+ "//mediapipe/calculators/util:alignment_points_to_rects_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
+ "//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:visibility_smoothing_calculator",
"//mediapipe/framework/tool:switch_container",
],
diff --git a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt
index 6f777ed5e..2560dda79 100644
--- a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt
+++ b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt
@@ -29,6 +29,29 @@ output_stream: "FILTERED_NORM_LANDMARKS:filtered_landmarks"
# Filtered auxiliary set of normalized landmarks. (NormalizedRect)
output_stream: "FILTERED_AUX_NORM_LANDMARKS:filtered_aux_landmarks"
+# Converts landmarks to a detection that tightly encloses all landmarks.
+node {
+ calculator: "LandmarksToDetectionCalculator"
+ input_stream: "NORM_LANDMARKS:aux_landmarks"
+ output_stream: "DETECTION:aux_detection"
+}
+
+# Converts detection into a rectangle based on center and scale alignment
+# points.
+node {
+ calculator: "AlignmentPointsRectsCalculator"
+ input_stream: "DETECTION:aux_detection"
+ input_stream: "IMAGE_SIZE:image_size"
+ output_stream: "NORM_RECT:roi"
+ options: {
+ [mediapipe.DetectionsToRectsCalculatorOptions.ext] {
+ rotation_vector_start_keypoint_index: 0
+ rotation_vector_end_keypoint_index: 1
+ rotation_vector_target_angle_degrees: 90
+ }
+ }
+}
+
# Smoothes pose landmark visibilities to reduce jitter.
node {
calculator: "SwitchContainer"
@@ -66,6 +89,7 @@ node {
input_side_packet: "ENABLE:enable"
input_stream: "NORM_LANDMARKS:filtered_visibility"
input_stream: "IMAGE_SIZE:image_size"
+ input_stream: "OBJECT_SCALE_ROI:roi"
output_stream: "NORM_FILTERED_LANDMARKS:filtered_landmarks"
options: {
[mediapipe.SwitchContainerOptions.ext] {
@@ -83,12 +107,12 @@ node {
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
one_euro_filter {
- # Min cutoff 0.1 results into ~ 0.02 alpha in landmark EMA filter
+ # Min cutoff 0.1 results into ~0.01 alpha in landmark EMA filter
# when landmark is static.
- min_cutoff: 0.1
- # Beta 40.0 in combintation with min_cutoff 0.1 results into ~0.8
- # alpha in landmark EMA filter when landmark is moving fast.
- beta: 40.0
+ min_cutoff: 0.05
+ # Beta 80.0 in combintation with min_cutoff 0.05 results into
+ # ~0.94 alpha in landmark EMA filter when landmark is moving fast.
+ beta: 80.0
# Derivative cutoff 1.0 results into ~0.17 alpha in landmark
# velocity EMA filter.
derivate_cutoff: 1.0
@@ -119,6 +143,7 @@ node {
calculator: "LandmarksSmoothingCalculator"
input_stream: "NORM_LANDMARKS:filtered_aux_visibility"
input_stream: "IMAGE_SIZE:image_size"
+ input_stream: "OBJECT_SCALE_ROI:roi"
output_stream: "NORM_FILTERED_LANDMARKS:filtered_aux_landmarks"
options: {
[mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
@@ -127,12 +152,12 @@ node {
# object is not moving but responsive enough in case of sudden
# movements.
one_euro_filter {
- # Min cutoff 0.01 results into ~ 0.002 alpha in landmark EMA
+ # Min cutoff 0.01 results into ~0.002 alpha in landmark EMA
# filter when landmark is static.
min_cutoff: 0.01
- # Beta 1.0 in combintation with min_cutoff 0.01 results into ~0.2
+ # Beta 10.0 in combintation with min_cutoff 0.01 results into ~0.68
# alpha in landmark EMA filter when landmark is moving fast.
- beta: 1.0
+ beta: 10.0
# Derivative cutoff 1.0 results into ~0.17 alpha in landmark
# velocity EMA filter.
derivate_cutoff: 1.0
diff --git a/mediapipe/modules/selfie_segmentation/BUILD b/mediapipe/modules/selfie_segmentation/BUILD
new file mode 100644
index 000000000..f2babd7c5
--- /dev/null
+++ b/mediapipe/modules/selfie_segmentation/BUILD
@@ -0,0 +1,73 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+ "//mediapipe/framework/tool:mediapipe_graph.bzl",
+ "mediapipe_simple_subgraph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_simple_subgraph(
+ name = "selfie_segmentation_model_loader",
+ graph = "selfie_segmentation_model_loader.pbtxt",
+ register_as = "SelfieSegmentationModelLoader",
+ deps = [
+ "//mediapipe/calculators/core:constant_side_packet_calculator",
+ "//mediapipe/calculators/tflite:tflite_model_calculator",
+ "//mediapipe/calculators/util:local_file_contents_calculator",
+ "//mediapipe/framework/tool:switch_container",
+ ],
+)
+
+mediapipe_simple_subgraph(
+ name = "selfie_segmentation_cpu",
+ graph = "selfie_segmentation_cpu.pbtxt",
+ register_as = "SelfieSegmentationCpu",
+ deps = [
+ ":selfie_segmentation_model_loader",
+ "//mediapipe/calculators/image:image_properties_calculator",
+ "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+ "//mediapipe/calculators/tensor:inference_calculator",
+ "//mediapipe/calculators/tensor:tensors_to_segmentation_calculator",
+ "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
+ "//mediapipe/calculators/util:from_image_calculator",
+ "//mediapipe/framework/tool:switch_container",
+ ],
+)
+
+mediapipe_simple_subgraph(
+ name = "selfie_segmentation_gpu",
+ graph = "selfie_segmentation_gpu.pbtxt",
+ register_as = "SelfieSegmentationGpu",
+ deps = [
+ ":selfie_segmentation_model_loader",
+ "//mediapipe/calculators/image:image_properties_calculator",
+ "//mediapipe/calculators/tensor:image_to_tensor_calculator",
+ "//mediapipe/calculators/tensor:inference_calculator",
+ "//mediapipe/calculators/tensor:tensors_to_segmentation_calculator",
+ "//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
+ "//mediapipe/calculators/util:from_image_calculator",
+ "//mediapipe/framework/tool:switch_container",
+ ],
+)
+
+exports_files(
+ srcs = [
+ "selfie_segmentation.tflite",
+ "selfie_segmentation_landscape.tflite",
+ ],
+)
diff --git a/mediapipe/modules/selfie_segmentation/README.md b/mediapipe/modules/selfie_segmentation/README.md
new file mode 100644
index 000000000..cd6c5e044
--- /dev/null
+++ b/mediapipe/modules/selfie_segmentation/README.md
@@ -0,0 +1,6 @@
+# selfie_segmentation
+
+Subgraphs|Details
+:--- | :---
+[`SelfieSegmentationCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation_cpu.pbtxt)| Segments the person from background in a selfie image. (CPU input, and inference is executed on CPU.)
+[`SelfieSegmentationGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation_gpu.pbtxt)| Segments the person from background in a selfie image. (GPU input, and inference is executed on GPU.)
diff --git a/mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite b/mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite
new file mode 100644
index 000000000..374c0720d
Binary files /dev/null and b/mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite differ
diff --git a/mediapipe/modules/selfie_segmentation/selfie_segmentation_cpu.pbtxt b/mediapipe/modules/selfie_segmentation/selfie_segmentation_cpu.pbtxt
new file mode 100644
index 000000000..550cee906
--- /dev/null
+++ b/mediapipe/modules/selfie_segmentation/selfie_segmentation_cpu.pbtxt
@@ -0,0 +1,131 @@
+# MediaPipe graph to perform selfie segmentation. (CPU input, and all processing
+# and inference are also performed on CPU)
+#
+# It is required that "selfie_segmentation.tflite" or
+# "selfie_segmentation_landscape.tflite" is available at
+# "mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite"
+# or
+# "mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite"
+# path respectively during execution, depending on the specification in the
+# MODEL_SELECTION input side packet.
+#
+# EXAMPLE:
+# node {
+# calculator: "SelfieSegmentationCpu"
+# input_side_packet: "MODEL_SELECTION:model_selection"
+# input_stream: "IMAGE:image"
+# output_stream: "SEGMENTATION_MASK:segmentation_mask"
+# }
+
+type: "SelfieSegmentationCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:image"
+
+# An integer 0 or 1. Use 0 to select a general-purpose model (operating on a
+# 256x256 tensor), and 1 to select a model (operating on a 256x144 tensor) more
+# optimized for landscape images. If unspecified, functions as set to 0. (int)
+input_side_packet: "MODEL_SELECTION:model_selection"
+
+# Segmentation mask. (ImageFrame in ImageFormat::VEC32F1)
+output_stream: "SEGMENTATION_MASK:segmentation_mask"
+
+# Resizes the input image into a tensor with a dimension desired by the model.
+node {
+ calculator: "SwitchContainer"
+ input_side_packet: "SELECT:model_selection"
+ input_stream: "IMAGE:image"
+ output_stream: "TENSORS:input_tensors"
+ options: {
+ [mediapipe.SwitchContainerOptions.ext] {
+ select: 0
+ contained_node: {
+ calculator: "ImageToTensorCalculator"
+ options: {
+ [mediapipe.ImageToTensorCalculatorOptions.ext] {
+ output_tensor_width: 256
+ output_tensor_height: 256
+ keep_aspect_ratio: false
+ output_tensor_float_range {
+ min: 0.0
+ max: 1.0
+ }
+ border_mode: BORDER_ZERO
+ }
+ }
+ }
+ contained_node: {
+ calculator: "ImageToTensorCalculator"
+ options: {
+ [mediapipe.ImageToTensorCalculatorOptions.ext] {
+ output_tensor_width: 256
+ output_tensor_height: 144
+ keep_aspect_ratio: false
+ output_tensor_float_range {
+ min: 0.0
+ max: 1.0
+ }
+ border_mode: BORDER_ZERO
+ }
+ }
+ }
+ }
+ }
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+ calculator: "TfLiteCustomOpResolverCalculator"
+ output_side_packet: "op_resolver"
+}
+
+# Loads the selfie segmentation TF Lite model.
+node {
+ calculator: "SelfieSegmentationModelLoader"
+ input_side_packet: "MODEL_SELECTION:model_selection"
+ output_side_packet: "MODEL:model"
+}
+
+# Runs model inference on CPU.
+node {
+ calculator: "InferenceCalculator"
+ input_stream: "TENSORS:input_tensors"
+ output_stream: "TENSORS:output_tensors"
+ input_side_packet: "MODEL:model"
+ input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
+ options: {
+ [mediapipe.InferenceCalculatorOptions.ext] {
+ delegate { xnnpack {} }
+ }
+ #
+ }
+}
+
+# Retrieves the size of the input image.
+node {
+ calculator: "ImagePropertiesCalculator"
+ input_stream: "IMAGE_CPU:image"
+ output_stream: "SIZE:input_size"
+}
+
+# Processes the output tensors into a segmentation mask that has the same size
+# as the input image into the graph.
+node {
+ calculator: "TensorsToSegmentationCalculator"
+ input_stream: "TENSORS:output_tensors"
+ input_stream: "OUTPUT_SIZE:input_size"
+ output_stream: "MASK:mask_image"
+ options: {
+ [mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
+ activation: NONE
+ }
+ }
+}
+
+# Converts the incoming Image into the corresponding ImageFrame type.
+node: {
+ calculator: "FromImageCalculator"
+ input_stream: "IMAGE:mask_image"
+ output_stream: "IMAGE_CPU:segmentation_mask"
+}
diff --git a/mediapipe/modules/selfie_segmentation/selfie_segmentation_gpu.pbtxt b/mediapipe/modules/selfie_segmentation/selfie_segmentation_gpu.pbtxt
new file mode 100644
index 000000000..5f9e55eb7
--- /dev/null
+++ b/mediapipe/modules/selfie_segmentation/selfie_segmentation_gpu.pbtxt
@@ -0,0 +1,133 @@
+# MediaPipe graph to perform selfie segmentation. (GPU input, and all processing
+# and inference are also performed on GPU)
+#
+# It is required that "selfie_segmentation.tflite" or
+# "selfie_segmentation_landscape.tflite" is available at
+# "mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite"
+# or
+# "mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite"
+# path respectively during execution, depending on the specification in the
+# MODEL_SELECTION input side packet.
+#
+# EXAMPLE:
+# node {
+# calculator: "SelfieSegmentationGpu"
+# input_side_packet: "MODEL_SELECTION:model_selection"
+# input_stream: "IMAGE:image"
+# output_stream: "SEGMENTATION_MASK:segmentation_mask"
+# }
+
+type: "SelfieSegmentationGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:image"
+
+# An integer 0 or 1. Use 0 to select a general-purpose model (operating on a
+# 256x256 tensor), and 1 to select a model (operating on a 256x144 tensor) more
+# optimized for landscape images. If unspecified, functions as set to 0. (int)
+input_side_packet: "MODEL_SELECTION:model_selection"
+
+# Segmentation mask. (GpuBuffer in RGBA, with the same mask values in R and A)
+output_stream: "SEGMENTATION_MASK:segmentation_mask"
+
+# Resizes the input image into a tensor with a dimension desired by the model.
+node {
+ calculator: "SwitchContainer"
+ input_side_packet: "SELECT:model_selection"
+ input_stream: "IMAGE_GPU:image"
+ output_stream: "TENSORS:input_tensors"
+ options: {
+ [mediapipe.SwitchContainerOptions.ext] {
+ select: 0
+ contained_node: {
+ calculator: "ImageToTensorCalculator"
+ options: {
+ [mediapipe.ImageToTensorCalculatorOptions.ext] {
+ output_tensor_width: 256
+ output_tensor_height: 256
+ keep_aspect_ratio: false
+ output_tensor_float_range {
+ min: 0.0
+ max: 1.0
+ }
+ border_mode: BORDER_ZERO
+ gpu_origin: TOP_LEFT
+ }
+ }
+ }
+ contained_node: {
+ calculator: "ImageToTensorCalculator"
+ options: {
+ [mediapipe.ImageToTensorCalculatorOptions.ext] {
+ output_tensor_width: 256
+ output_tensor_height: 144
+ keep_aspect_ratio: false
+ output_tensor_float_range {
+ min: 0.0
+ max: 1.0
+ }
+ border_mode: BORDER_ZERO
+ gpu_origin: TOP_LEFT
+ }
+ }
+ }
+ }
+ }
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+ calculator: "TfLiteCustomOpResolverCalculator"
+ output_side_packet: "op_resolver"
+ options: {
+ [mediapipe.TfLiteCustomOpResolverCalculatorOptions.ext] {
+ use_gpu: true
+ }
+ }
+}
+
+# Loads the selfie segmentation TF Lite model.
+node {
+ calculator: "SelfieSegmentationModelLoader"
+ input_side_packet: "MODEL_SELECTION:model_selection"
+ output_side_packet: "MODEL:model"
+}
+
+# Runs model inference on GPU.
+node {
+ calculator: "InferenceCalculator"
+ input_stream: "TENSORS:input_tensors"
+ output_stream: "TENSORS:output_tensors"
+ input_side_packet: "MODEL:model"
+ input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
+}
+
+# Retrieves the size of the input image.
+node {
+ calculator: "ImagePropertiesCalculator"
+ input_stream: "IMAGE_GPU:image"
+ output_stream: "SIZE:input_size"
+}
+
+# Processes the output tensors into a segmentation mask that has the same size
+# as the input image into the graph.
+node {
+ calculator: "TensorsToSegmentationCalculator"
+ input_stream: "TENSORS:output_tensors"
+ input_stream: "OUTPUT_SIZE:input_size"
+ output_stream: "MASK:mask_image"
+ options: {
+ [mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
+ activation: NONE
+ gpu_origin: TOP_LEFT
+ }
+ }
+}
+
+# Converts the incoming Image into the corresponding GpuBuffer type.
+node: {
+ calculator: "FromImageCalculator"
+ input_stream: "IMAGE:mask_image"
+ output_stream: "IMAGE_GPU:segmentation_mask"
+}
diff --git a/mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite b/mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite
new file mode 100755
index 000000000..4ea3f8a10
Binary files /dev/null and b/mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite differ
diff --git a/mediapipe/modules/selfie_segmentation/selfie_segmentation_model_loader.pbtxt b/mediapipe/modules/selfie_segmentation/selfie_segmentation_model_loader.pbtxt
new file mode 100644
index 000000000..39495f80d
--- /dev/null
+++ b/mediapipe/modules/selfie_segmentation/selfie_segmentation_model_loader.pbtxt
@@ -0,0 +1,63 @@
+# MediaPipe graph to load a selected selfie segmentation TF Lite model.
+
+type: "SelfieSegmentationModelLoader"
+
+# An integer 0 or 1. Use 0 to select a general-purpose model (operating on a
+# 256x256 tensor), and 1 to select a model (operating on a 256x144 tensor) more
+# optimized for landscape images. If unspecified, functions as set to 0. (int)
+input_side_packet: "MODEL_SELECTION:model_selection"
+
+# TF Lite model represented as a FlatBuffer.
+# (std::unique_ptr>)
+output_side_packet: "MODEL:model"
+
+# Determines path to the desired pose landmark model file.
+node {
+ calculator: "SwitchContainer"
+ input_side_packet: "SELECT:model_selection"
+ output_side_packet: "PACKET:model_path"
+ options: {
+ [mediapipe.SwitchContainerOptions.ext] {
+ select: 0
+ contained_node: {
+ calculator: "ConstantSidePacketCalculator"
+ options: {
+ [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+ packet {
+ string_value: "mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite"
+ }
+ }
+ }
+ }
+ contained_node: {
+ calculator: "ConstantSidePacketCalculator"
+ options: {
+ [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+ packet {
+ string_value: "mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite"
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+# Loads the file in the specified path into a blob.
+node {
+ calculator: "LocalFileContentsCalculator"
+ input_side_packet: "FILE_PATH:model_path"
+ output_side_packet: "CONTENTS:model_blob"
+ options: {
+ [mediapipe.LocalFileContentsCalculatorOptions.ext]: {
+ text_mode: false
+ }
+ }
+}
+
+# Converts the input blob into a TF Lite model.
+node {
+ calculator: "TfLiteModelCalculator"
+ input_side_packet: "MODEL_BLOB:model_blob"
+ output_side_packet: "MODEL:model"
+}
diff --git a/mediapipe/opensource_only/ISSUE_TEMPLATE/30-bug-issue.md b/mediapipe/opensource_only/ISSUE_TEMPLATE/30-bug-issue.md
new file mode 100644
index 000000000..f31f3649f
--- /dev/null
+++ b/mediapipe/opensource_only/ISSUE_TEMPLATE/30-bug-issue.md
@@ -0,0 +1,26 @@
+Please make sure that this is a bug and also refer to the [troubleshooting](https://google.github.io/mediapipe/getting_started/troubleshooting.html), FAQ documentation before raising any issues.
+
+**System information** (Please provide as much relevant information as possible)
+
+- Have I written custom code (as opposed to using a stock example script provided in MediaPipe):
+- OS Platform and Distribution (e.g., Linux Ubuntu 16.04, Android 11, iOS 14.4):
+- Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device:
+- Browser and version (e.g. Google Chrome, Safari) if the issue happens on browser:
+- Programming Language and version ( e.g. C++, Python, Java):
+- [MediaPipe version](https://github.com/google/mediapipe/releases):
+- Bazel version (if compiling from source):
+- Solution ( e.g. FaceMesh, Pose, Holistic ):
+- Android Studio, NDK, SDK versions (if issue is related to building in Android environment):
+- Xcode & Tulsi version (if issue is related to building for iOS):
+
+**Describe the current behavior:**
+
+**Describe the expected behavior:**
+
+**Standalone code to reproduce the issue:**
+Provide a reproducible test case that is the bare minimum necessary to replicate the problem. If possible, please share a link to Colab/repo link /any notebook:
+
+**Other info / Complete Logs :**
+ Include any logs or source code that would be helpful to
+diagnose the problem. If including tracebacks, please include the full
+traceback. Large logs and files should be attached
diff --git a/mediapipe/opensource_only/ISSUE_TEMPLATE/40-feature-request.md b/mediapipe/opensource_only/ISSUE_TEMPLATE/40-feature-request.md
new file mode 100644
index 000000000..2da72f3b1
--- /dev/null
+++ b/mediapipe/opensource_only/ISSUE_TEMPLATE/40-feature-request.md
@@ -0,0 +1,18 @@
+Please make sure that this is a feature request.
+
+**System information** (Please provide as much relevant information as possible)
+
+- MediaPipe Solution (you are using):
+- Programming language : C++/typescript/Python/Objective C/Android Java
+- Are you willing to contribute it (Yes/No):
+
+
+**Describe the feature and the current behavior/state:**
+
+**Will this change the current api? How?**
+
+**Who will benefit with this feature?**
+
+**Please specify the use cases for this feature:**
+
+**Any Other info:**
diff --git a/mediapipe/opensource_only/ISSUE_TEMPLATE/50-other-issues.md b/mediapipe/opensource_only/ISSUE_TEMPLATE/50-other-issues.md
new file mode 100644
index 000000000..9e094dd9c
--- /dev/null
+++ b/mediapipe/opensource_only/ISSUE_TEMPLATE/50-other-issues.md
@@ -0,0 +1,8 @@
+This template is for miscellaneous issues not covered by the other issue categories
+
+For questions on how to work with MediaPipe, or support for problems that are not verified bugs in MediaPipe, please go to [StackOverflow](https://stackoverflow.com/questions/tagged/mediapipe) and [Slack](https://mediapipe.page.link/joinslack) communities.
+
+If you are reporting a vulnerability, please use the [dedicated reporting process](https://github.com/google/mediapipe/security).
+
+For high-level discussions about MediaPipe, please post to discuss@mediapipe.org, for questions about the development or internal workings of MediaPipe, or if you would like to know how to contribute to MediaPipe, please post to developers@mediapipe.org.
+
diff --git a/mediapipe/python/BUILD b/mediapipe/python/BUILD
index 08a299589..11fe45835 100644
--- a/mediapipe/python/BUILD
+++ b/mediapipe/python/BUILD
@@ -72,5 +72,6 @@ cc_library(
"//mediapipe/modules/pose_detection:pose_detection_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_by_roi_cpu",
"//mediapipe/modules/pose_landmark:pose_landmark_cpu",
+ "//mediapipe/modules/selfie_segmentation:selfie_segmentation_cpu",
],
)
diff --git a/mediapipe/python/solutions/__init__.py b/mediapipe/python/solutions/__init__.py
index 8cd9af327..fc0686d22 100644
--- a/mediapipe/python/solutions/__init__.py
+++ b/mediapipe/python/solutions/__init__.py
@@ -21,3 +21,4 @@ import mediapipe.python.solutions.hands
import mediapipe.python.solutions.holistic
import mediapipe.python.solutions.objectron
import mediapipe.python.solutions.pose
+import mediapipe.python.solutions.selfie_segmentation
diff --git a/mediapipe/python/solutions/drawing_utils.py b/mediapipe/python/solutions/drawing_utils.py
index 40259153e..2e0fd9971 100644
--- a/mediapipe/python/solutions/drawing_utils.py
+++ b/mediapipe/python/solutions/drawing_utils.py
@@ -15,7 +15,7 @@
"""MediaPipe solution drawing utils."""
import math
-from typing import List, Tuple, Union
+from typing import List, Optional, Tuple, Union
import cv2
import dataclasses
@@ -116,7 +116,7 @@ def draw_detection(
def draw_landmarks(
image: np.ndarray,
landmark_list: landmark_pb2.NormalizedLandmarkList,
- connections: List[Tuple[int, int]] = None,
+ connections: Optional[List[Tuple[int, int]]] = None,
landmark_drawing_spec: DrawingSpec = DrawingSpec(color=RED_COLOR),
connection_drawing_spec: DrawingSpec = DrawingSpec()):
"""Draws the landmarks and the connections on the image.
diff --git a/mediapipe/python/solutions/face_detection_test.py b/mediapipe/python/solutions/face_detection_test.py
index 25f5b33fd..f4185ea46 100644
--- a/mediapipe/python/solutions/face_detection_test.py
+++ b/mediapipe/python/solutions/face_detection_test.py
@@ -56,7 +56,8 @@ class FaceDetectionTest(absltest.TestCase):
self.assertIsNone(results.detections)
def test_face(self):
- image_path = os.path.join(os.path.dirname(__file__), 'testdata/face.jpg')
+ image_path = os.path.join(os.path.dirname(__file__),
+ 'testdata/portrait.jpg')
image = cv2.imread(image_path)
with mp_faces.FaceDetection(min_detection_confidence=0.5) as faces:
for idx in range(5):
diff --git a/mediapipe/python/solutions/face_mesh_test.py b/mediapipe/python/solutions/face_mesh_test.py
index cf112044d..2d8503872 100644
--- a/mediapipe/python/solutions/face_mesh_test.py
+++ b/mediapipe/python/solutions/face_mesh_test.py
@@ -96,7 +96,8 @@ class FaceMeshTest(parameterized.TestCase):
@parameterized.named_parameters(('static_image_mode', True, 1),
('video_mode', False, 5))
def test_face(self, static_image_mode: bool, num_frames: int):
- image_path = os.path.join(os.path.dirname(__file__), 'testdata/face.jpg')
+ image_path = os.path.join(os.path.dirname(__file__),
+ 'testdata/portrait.jpg')
image = cv2.imread(image_path)
with mp_faces.FaceMesh(
static_image_mode=static_image_mode,
diff --git a/mediapipe/python/solutions/pose_test.py b/mediapipe/python/solutions/pose_test.py
index 5022514ea..2fb199919 100644
--- a/mediapipe/python/solutions/pose_test.py
+++ b/mediapipe/python/solutions/pose_test.py
@@ -30,18 +30,18 @@ from mediapipe.python.solutions import drawing_utils as mp_drawing
from mediapipe.python.solutions import pose as mp_pose
TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
-DIFF_THRESHOLD = 30 # pixels
-EXPECTED_POSE_LANDMARKS = np.array([[460, 287], [469, 277], [472, 276],
- [475, 276], [464, 277], [463, 277],
- [463, 276], [492, 277], [472, 277],
- [471, 295], [465, 295], [542, 323],
- [448, 318], [619, 319], [372, 313],
- [695, 316], [296, 308], [717, 313],
- [273, 304], [718, 304], [280, 298],
- [709, 307], [289, 303], [521, 470],
- [459, 466], [626, 533], [364, 500],
- [704, 616], [347, 614], [710, 631],
- [357, 633], [737, 625], [306, 639]])
+DIFF_THRESHOLD = 15 # pixels
+EXPECTED_POSE_LANDMARKS = np.array([[460, 283], [467, 273], [471, 273],
+ [474, 273], [465, 273], [465, 273],
+ [466, 273], [491, 277], [480, 277],
+ [470, 294], [465, 294], [545, 319],
+ [453, 329], [622, 323], [375, 316],
+ [696, 316], [299, 307], [719, 316],
+ [278, 306], [721, 311], [274, 304],
+ [713, 313], [283, 306], [520, 476],
+ [467, 471], [612, 550], [358, 490],
+ [701, 613], [349, 611], [709, 624],
+ [363, 630], [730, 633], [303, 628]])
class PoseTest(parameterized.TestCase):
diff --git a/mediapipe/python/solutions/selfie_segmentation.py b/mediapipe/python/solutions/selfie_segmentation.py
new file mode 100644
index 000000000..8aa07569c
--- /dev/null
+++ b/mediapipe/python/solutions/selfie_segmentation.py
@@ -0,0 +1,76 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MediaPipe Selfie Segmentation."""
+
+from typing import NamedTuple
+
+import numpy as np
+# The following imports are needed because python pb2 silently discards
+# unknown protobuf fields.
+# pylint: disable=unused-import
+from mediapipe.calculators.core import constant_side_packet_calculator_pb2
+from mediapipe.calculators.tensor import image_to_tensor_calculator_pb2
+from mediapipe.calculators.tensor import inference_calculator_pb2
+from mediapipe.calculators.tensor import tensors_to_segmentation_calculator_pb2
+from mediapipe.calculators.util import local_file_contents_calculator_pb2
+from mediapipe.framework.tool import switch_container_pb2
+# pylint: enable=unused-import
+
+from mediapipe.python.solution_base import SolutionBase
+
+BINARYPB_FILE_PATH = 'mediapipe/modules/selfie_segmentation/selfie_segmentation_cpu.binarypb'
+
+
+class SelfieSegmentation(SolutionBase):
+ """MediaPipe Selfie Segmentation.
+
+ MediaPipe Selfie Segmentation processes an RGB image and returns a
+ segmentation mask.
+
+ Please refer to
+ https://solutions.mediapipe.dev/selfie_segmentation#python-solution-api for
+ usage examples.
+ """
+
+ def __init__(self, model_selection=0):
+ """Initializes a MediaPipe Selfie Segmentation object.
+
+ Args:
+ model_selection: 0 or 1. 0 to select a general-purpose model, and 1 to
+ select a model more optimized for landscape images. See details in
+ https://solutions.mediapipe.dev/selfie_segmentation#model_selection.
+ """
+ super().__init__(
+ binary_graph_path=BINARYPB_FILE_PATH,
+ side_inputs={
+ 'model_selection': model_selection,
+ },
+ outputs=['segmentation_mask'])
+
+ def process(self, image: np.ndarray) -> NamedTuple:
+ """Processes an RGB image and returns a segmentation mask.
+
+ Args:
+ image: An RGB image represented as a numpy ndarray.
+
+ Raises:
+ RuntimeError: If the underlying graph throws any error.
+ ValueError: If the input image is not three channel RGB.
+
+ Returns:
+ A NamedTuple object with a "segmentation_mask" field that contains a float
+ type 2d np array representing the mask.
+ """
+
+ return super().process(input_data={'image': image})
diff --git a/mediapipe/python/solutions/selfie_segmentation_test.py b/mediapipe/python/solutions/selfie_segmentation_test.py
new file mode 100644
index 000000000..3dba08876
--- /dev/null
+++ b/mediapipe/python/solutions/selfie_segmentation_test.py
@@ -0,0 +1,68 @@
+# Copyright 2021 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for mediapipe.python.solutions.selfie_segmentation."""
+
+import os
+
+from absl.testing import absltest
+from absl.testing import parameterized
+import cv2
+import numpy as np
+
+# resources dependency
+# undeclared dependency
+from mediapipe.python.solutions import selfie_segmentation as mp_selfie_segmentation
+
+TEST_IMAGE_PATH = 'mediapipe/python/solutions/testdata'
+
+
+class SelfieSegmentationTest(parameterized.TestCase):
+
+ def _draw(self, frame: np.ndarray, mask: np.ndarray):
+ frame = np.minimum(frame, np.stack((mask,) * 3, axis=-1))
+ path = os.path.join(tempfile.gettempdir(), self.id().split('.')[-1] + '.png')
+ cv2.imwrite(path, frame)
+
+ def test_invalid_image_shape(self):
+ with mp_selfie_segmentation.SelfieSegmentation() as selfie_segmentation:
+ with self.assertRaisesRegex(
+ ValueError, 'Input image must contain three channel rgb data.'):
+ selfie_segmentation.process(
+ np.arange(36, dtype=np.uint8).reshape(3, 3, 4))
+
+ def test_blank_image(self):
+ with mp_selfie_segmentation.SelfieSegmentation() as selfie_segmentation:
+ image = np.zeros([100, 100, 3], dtype=np.uint8)
+ image.fill(255)
+ results = selfie_segmentation.process(image)
+ normalized_segmentation_mask = (results.segmentation_mask *
+ 255).astype(int)
+ self.assertLess(np.amax(normalized_segmentation_mask), 1)
+
+ @parameterized.named_parameters(('general', 0), ('landscape', 1))
+ def test_segmentation(self, model_selection):
+ image_path = os.path.join(os.path.dirname(__file__),
+ 'testdata/portrait.jpg')
+ image = cv2.imread(image_path)
+ with mp_selfie_segmentation.SelfieSegmentation(
+ model_selection=model_selection) as selfie_segmentation:
+ results = selfie_segmentation.process(
+ cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+ normalized_segmentation_mask = (results.segmentation_mask *
+ 255).astype(int)
+ self._draw(image.copy(), normalized_segmentation_mask)
+
+
+if __name__ == '__main__':
+ absltest.main()
diff --git a/mediapipe/util/resource_util.cc b/mediapipe/util/resource_util.cc
index 042d1e810..8f40154a0 100644
--- a/mediapipe/util/resource_util.cc
+++ b/mediapipe/util/resource_util.cc
@@ -31,10 +31,10 @@ ResourceProviderFn resource_provider_ = nullptr;
absl::Status GetResourceContents(const std::string& path, std::string* output,
bool read_as_binary) {
- if (resource_provider_ == nullptr || !resource_provider_(path, output).ok()) {
- return internal::DefaultGetResourceContents(path, output, read_as_binary);
+ if (resource_provider_) {
+ return resource_provider_(path, output);
}
- return absl::OkStatus();
+ return internal::DefaultGetResourceContents(path, output, read_as_binary);
}
void SetCustomGlobalResourceProvider(ResourceProviderFn fn) {
diff --git a/mediapipe/util/resource_util_android.cc b/mediapipe/util/resource_util_android.cc
index b7589d8fe..b18354d5f 100644
--- a/mediapipe/util/resource_util_android.cc
+++ b/mediapipe/util/resource_util_android.cc
@@ -51,7 +51,9 @@ absl::Status DefaultGetResourceContents(const std::string& path,
// Try the test environment.
absl::string_view workspace = "mediapipe";
- auto test_path = file::JoinPath(std::getenv("TEST_SRCDIR"), workspace, path);
+ const char* test_srcdir = std::getenv("TEST_SRCDIR");
+ auto test_path =
+ file::JoinPath(test_srcdir ? test_srcdir : "", workspace, path);
if (file::Exists(test_path).ok()) {
return file::GetContents(path, output, file::Defaults());
}
diff --git a/mediapipe/util/resource_util_apple.cc b/mediapipe/util/resource_util_apple.cc
index 428018ee4..5fb66c24c 100644
--- a/mediapipe/util/resource_util_apple.cc
+++ b/mediapipe/util/resource_util_apple.cc
@@ -88,8 +88,9 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) {
// Try the test environment.
{
absl::string_view workspace = "mediapipe";
+ const char* test_srcdir = std::getenv("TEST_SRCDIR");
auto test_path =
- file::JoinPath(std::getenv("TEST_SRCDIR"), workspace, path);
+ file::JoinPath(test_srcdir ? test_srcdir : "", workspace, path);
if ([[NSFileManager defaultManager]
fileExistsAtPath:[NSString
stringWithUTF8String:test_path.c_str()]]) {
diff --git a/mediapipe/util/tflite/BUILD b/mediapipe/util/tflite/BUILD
index 4d66bbe21..8c9687723 100644
--- a/mediapipe/util/tflite/BUILD
+++ b/mediapipe/util/tflite/BUILD
@@ -81,7 +81,7 @@ cc_library(
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/delegates/gpu:api",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:model",
- "@org_tensorflow//tensorflow/lite/delegates/gpu/common/testing:tflite_model_reader",
+ "@org_tensorflow//tensorflow/lite/delegates/gpu/common:model_builder",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:api2",
],
"//mediapipe:android": [
@@ -93,7 +93,7 @@ cc_library(
"@org_tensorflow//tensorflow/lite/delegates/gpu:api",
"@org_tensorflow//tensorflow/lite/delegates/gpu/cl:api",
"@org_tensorflow//tensorflow/lite/delegates/gpu/common:model",
- "@org_tensorflow//tensorflow/lite/delegates/gpu/common/testing:tflite_model_reader",
+ "@org_tensorflow//tensorflow/lite/delegates/gpu/common:model_builder",
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:api2",
],
}) + ["@org_tensorflow//tensorflow/lite/core/api"],
diff --git a/mediapipe/util/tflite/operations/BUILD b/mediapipe/util/tflite/operations/BUILD
index a12bbf4a1..1d4a76d12 100644
--- a/mediapipe/util/tflite/operations/BUILD
+++ b/mediapipe/util/tflite/operations/BUILD
@@ -17,6 +17,8 @@ licenses(["notice"])
package(default_visibility = [
"//mediapipe:__subpackages__",
+ # For automated benchmarking of Camera models by TFLite team.
+ "//learning/brain/models/app_benchmarks/camera_models:__subpackages__",
])
cc_library(
diff --git a/mediapipe/util/tflite/tflite_gpu_runner.cc b/mediapipe/util/tflite/tflite_gpu_runner.cc
index c77c09524..ef236bf93 100644
--- a/mediapipe/util/tflite/tflite_gpu_runner.cc
+++ b/mediapipe/util/tflite/tflite_gpu_runner.cc
@@ -27,6 +27,7 @@
#include "tensorflow/lite/core/api/op_resolver.h"
#include "tensorflow/lite/delegates/gpu/api.h"
#include "tensorflow/lite/delegates/gpu/common/model.h"
+#include "tensorflow/lite/delegates/gpu/common/model_builder.h"
#include "tensorflow/lite/delegates/gpu/gl/api2.h"
#include "tensorflow/lite/model.h"
@@ -35,7 +36,6 @@
#ifdef __ANDROID__
#include "tensorflow/lite/delegates/gpu/cl/api.h"
#endif
-#include "tensorflow/lite/delegates/gpu/common/testing/tflite_model_reader.h"
namespace tflite {
namespace gpu {
diff --git a/mediapipe/util/tflite/tflite_model_loader.cc b/mediapipe/util/tflite/tflite_model_loader.cc
index a87d94bd6..abd0e7257 100644
--- a/mediapipe/util/tflite/tflite_model_loader.cc
+++ b/mediapipe/util/tflite/tflite_model_loader.cc
@@ -23,11 +23,31 @@ absl::StatusOr> TfLiteModelLoader::LoadFromPath(
const std::string& path) {
std::string model_path = path;
- ASSIGN_OR_RETURN(model_path, mediapipe::PathToResourceAsFile(model_path));
- auto model = tflite::FlatBufferModel::BuildFromFile(model_path.c_str());
+ std::string model_blob;
+ auto status_or_content =
+ mediapipe::GetResourceContents(model_path, &model_blob);
+ // TODO: get rid of manual resolving with PathToResourceAsFile
+ // as soon as it's incorporated into GetResourceContents.
+ if (!status_or_content.ok()) {
+ LOG(WARNING)
+ << "Trying to resolve path manually as GetResourceContents failed: "
+ << status_or_content.message();
+ ASSIGN_OR_RETURN(auto resolved_path,
+ mediapipe::PathToResourceAsFile(model_path));
+ MP_RETURN_IF_ERROR(
+ mediapipe::GetResourceContents(resolved_path, &model_blob));
+ }
+
+ auto model = tflite::FlatBufferModel::VerifyAndBuildFromBuffer(
+ model_blob.data(), model_blob.size());
RET_CHECK(model) << "Failed to load model from path " << model_path;
return api2::MakePacket(
- model.release(), [](tflite::FlatBufferModel* model) { delete model; });
+ model.release(),
+ [model_blob = std::move(model_blob)](tflite::FlatBufferModel* model) {
+ // It's required that model_blob is deleted only after
+ // model is deleted, hence capturing model_blob.
+ delete model;
+ });
}
} // namespace mediapipe
diff --git a/setup.py b/setup.py
index 81569b34d..31e5195bb 100644
--- a/setup.py
+++ b/setup.py
@@ -226,7 +226,8 @@ class BuildBinaryGraphs(build.build):
'face_landmark/face_landmark_front_cpu',
'hand_landmark/hand_landmark_tracking_cpu',
'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu',
- 'pose_landmark/pose_landmark_cpu'
+ 'pose_landmark/pose_landmark_cpu',
+ 'selfie_segmentation/selfie_segmentation_cpu'
]
for binary_graph in binary_graphs:
sys.stderr.write('generating binarypb: %s\n' %
@@ -379,12 +380,20 @@ class RemoveGenerated(clean.clean):
def run(self):
for pattern in [
- 'mediapipe/framework/**/*pb2.py', 'mediapipe/calculators/**/*pb2.py',
- 'mediapipe/gpu/**/*pb2.py', 'mediapipe/util/**/*pb2.py'
+ 'mediapipe/calculators/**/*pb2.py',
+ 'mediapipe/framework/**/*pb2.py',
+ 'mediapipe/gpu/**/*pb2.py',
+ 'mediapipe/modules/**/*pb2.py',
+ 'mediapipe/util/**/*pb2.py',
]:
for py_file in glob.glob(pattern, recursive=True):
sys.stderr.write('removing generated files: %s\n' % py_file)
os.remove(py_file)
+ init_py = os.path.join(
+ os.path.dirname(os.path.abspath(py_file)), '__init__.py')
+ if os.path.exists(init_py):
+ sys.stderr.write('removing __init__ file: %s\n' % init_py)
+ os.remove(init_py)
for binarypb_file in glob.glob(
'mediapipe/modules/**/*.binarypb', recursive=True):
sys.stderr.write('removing generated binary graphs: %s\n' % binarypb_file)
diff --git a/third_party/org_tensorflow_compatibility_fixes.diff b/third_party/org_tensorflow_compatibility_fixes.diff
index 2846fcc80..46770e640 100644
--- a/third_party/org_tensorflow_compatibility_fixes.diff
+++ b/third_party/org_tensorflow_compatibility_fixes.diff
@@ -35,19 +35,6 @@ index ba50783765..5de5ea01f0 100644
#include
#include
#include
-diff --git a/tensorflow/lite/delegates/gpu/cl/serialization.fbs b/tensorflow/lite/delegates/gpu/cl/serialization.fbs
-index 67bd587162e..2a3c6bd30dc 100644
---- a/tensorflow/lite/delegates/gpu/cl/serialization.fbs
-+++ b/tensorflow/lite/delegates/gpu/cl/serialization.fbs
-@@ -12,7 +12,7 @@
- // See the License for the specific language governing permissions and
- // limitations under the License.
-
--include "tensorflow/lite/delegates/gpu/common/task/serialization_base.fbs";
-+include "../common/task/serialization_base.fbs";
-
- namespace tflite.gpu.cl.data;
-
diff --git a/third_party/eigen3/eigen_archive.BUILD b/third_party/eigen3/eigen_archive.BUILD
index dad592bec48..670017c2c0f 100644
--- a/third_party/eigen3/eigen_archive.BUILD