Project import generated by Copybara.

GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c

Dockerfile

@@ -54,7 +54,7 @@ RUN pip3 install tf_slim
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
-ARG BAZEL_VERSION=2.0.0
+ARG BAZEL_VERSION=3.0.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

README.md

@@ -89,7 +89,8 @@ run code search using
 
 ## Publications
 
-*   [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
+*   [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
+*   [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
     in Google Developers Blog
 *   [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
     in Google AI Blog

build_desktop_examples.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========================================================================
+#
+# Script to build/run all MediaPipe desktop example apps (with webcam input).
+#
+# To build and run all apps and store them in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir
+#   Omitting -d and the associated directory saves all generated apps in the
+#   current directory.
+# To build all apps and store them in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir -b
+#   Omitting -d and the associated directory saves all generated apps in the
+#   current directory.
+# To run all apps already stored in out_dir:
+#   $ ./build_ios_examples.sh -d out_dir -r
+#   Omitting -d and the associated directory assumes all apps are in the current
+#   directory.
+
+set -e
+
+out_dir="."
+build_only=false
+run_only=false
+app_dir="mediapipe/examples/desktop"
+bin_dir="bazel-bin"
+declare -a default_bazel_flags=(build -c opt --define MEDIAPIPE_DISABLE_GPU=1)
+
+while [[ -n $1 ]]; do
+  case $1 in
+    -d)
+      shift
+      out_dir=$1
+      ;;
+    -b)
+      build_only=true
+      ;;
+    -r)
+      run_only=true
+      ;;
+    *)
+      echo "Unsupported input argument $1."
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+echo "app_dir: $app_dir"
+echo "out_dir: $out_dir"
+
+declare -a bazel_flags
+
+apps="${app_dir}/*"
+for app in ${apps}; do
+  if [[ -d "${app}" ]]; then
+    target_name=${app##*/}
+    if [[ "${target_name}" == "autoflip" ||
+          "${target_name}" == "hello_world" ||
+          "${target_name}" == "media_sequence" ||
+          "${target_name}" == "template_matching" ||
+          "${target_name}" == "youtube8m" ]]; then
+      continue
+    fi
+    target="${app}:${target_name}_cpu"
+
+    echo "=== Target: ${target}"
+
+    if [[ $run_only == false ]]; then
+      bazel_flags=("${default_bazel_flags[@]}")
+      bazel_flags+=(${target})
+
+      bazel "${bazel_flags[@]}"
+      cp -f "${bin_dir}/${app}/"*"_cpu" "${out_dir}"
+    fi
+    if [[ $build_only == false ]]; then
+      if  [[ ${target_name} == "multi_hand_tracking" ]]; then
+        graph_name="hand_tracking/multi_hand_tracking"
+      elif  [[ ${target_name} == "object_tracking" ]]; then
+        graph_name="tracking/object_detection_tracking"
+      elif [[ ${target_name} == "upper_body_pose_tracking" ]]; then
+        graph_name="pose_tracking/upper_body_pose_tracking"
+      else
+        graph_name="${target_name}/${target_name}"
+      fi
+      if [[ ${target_name} == "iris_tracking" ||
+            ${target_name} == "upper_body_pose_tracking" ]]; then
+        graph_suffix="cpu"
+      else
+        graph_suffix="desktop_live"
+      fi
+      GLOG_logtostderr=1 "${out_dir}/${target_name}_cpu" \
+        --calculator_graph_config_file=mediapipe/graphs/"${graph_name}_${graph_suffix}.pbtxt"
+    fi
+  fi
+done

docs/index.md

@@ -89,7 +89,8 @@ run code search using
 
 ## Publications
 
-*   [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
+*   [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
+*   [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
     in Google Developers Blog
 *   [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
     in Google AI Blog

docs/solutions/face_detection.md

@@ -102,9 +102,4 @@ to cross-compile and run MediaPipe examples on the
     [BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs](https://arxiv.org/abs/1907.05047)
     ([presentation](https://docs.google.com/presentation/d/1YCtASfnYyZtH-41QvnW5iZxELFnf0MF-pPWSLGj8yjQ/present?slide=id.g5bc8aeffdd_1_0))
     ([poster](https://drive.google.com/file/d/1u6aB6wxDY7X2TmeUUKgFydulNtXkb3pu/view))
-*   For front-facing/selfie camera:
+*   [Models and model cards](./models.md#face_detection)
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
-    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
-*   For back-facing camera:
-    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
-*   [Model card](https://mediapipe.page.link/blazeface-mc)

docs/solutions/face_mesh.md

@@ -19,13 +19,18 @@ landmarks in real-time even on mobile devices. It employs machine learning (ML)
 to infer the 3D surface geometry, requiring only a single camera input without
 the need for a dedicated depth sensor. Utilizing lightweight model architectures
 together with GPU acceleration throughout the pipeline, the solution delivers
-real-time performance critical for live experiences. The core of the solution is
+real-time performance critical for live experiences.
-the same as what powers
+
-[YouTube Stories](https://youtube-creators.googleblog.com/2018/11/introducing-more-ways-to-share-your.html)'
+Additionally, the solution is bundled with the Face Geometry module that bridges
-creator effects, the
+the gap between the face landmark estimation and useful real-time augmented
-[Augmented Faces API in ARCore](https://developers.google.com/ar/develop/java/augmented-faces/)
+reality (AR) applications. It establishes a metric 3D space and uses the face
-and the
+landmark screen positions to estimate face geometry within that space. The face
-[ML Kit Face Contour Detection API](https://firebase.google.com/docs/ml-kit/face-detection-concepts#contours).
+geometry data consists of common 3D geometry primitives, including a face pose
+transformation matrix and a triangular face mesh. Under the hood, a lightweight
+statistical analysis method called
+[Procrustes Analysis](https://en.wikipedia.org/wiki/Procrustes_analysis) is
+employed to drive a robust, performant and portable logic. The analysis runs on
+CPU and has a minimal speed/memory footprint on top of the ML model inference.
 
 ![face_mesh_ar_effects.gif](../images/face_mesh_ar_effects.gif) |
 :-------------------------------------------------------------: |
@@ -67,15 +72,15 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-## Models
+### Models
 
-### Face Detection Model
+#### Face Detection Model
 
 The face detector is the same [BlazeFace](https://arxiv.org/abs/1907.05047)
 model used in [MediaPipe Face Detection](./face_detection.md). Please refer to
 [MediaPipe Face Detection](./face_detection.md) for details.
 
-### Face Landmark Model
+#### Face Landmark Model
 
 For 3D face landmarks we employed transfer learning and trained a network with
 several objectives: the network simultaneously predicts 3D landmark coordinates
@@ -98,7 +103,108 @@ You can find more information about the face landmark model in this
 
 ![face_mesh_android_gpu.gif](../images/mobile/face_mesh_android_gpu.gif)   |
 :------------------------------------------------------------------------: |
-*Fig 2. Output of MediaPipe Face Mesh: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
+*Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
+
+## Face Geometry Module
+
+The [Face Landmark Model](#face-landmark-model) performs a single-camera face landmark
+detection in the screen coordinate space: the X- and Y- coordinates are
+normalized screen coordinates, while the Z coordinate is relative and is scaled
+as the X coodinate under the
+[weak perspective projection camera model](https://en.wikipedia.org/wiki/3D_projection#Weak_perspective_projection).
+This format is well-suited for some applications, however it does not directly
+enable the full spectrum of augmented reality (AR) features like aligning a
+virtual 3D object with a detected face.
+
+The
+[Face Geometry module](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry)
+moves away from the screen coordinate space towards a metric 3D space and
+provides necessary primitives to handle a detected face as a regular 3D object.
+By design, you'll be able to use a perspective camera to project the final 3D
+scene back into the screen coordinate space with a guarantee that the face
+landmark positions are not changed.
+
+### Key Concepts
+
+#### Metric 3D Space
+
+The **Metric 3D space** established within the Face Geometry module is a
+right-handed orthonormal metric 3D coordinate space. Within the space, there is
+a **virtual perspective camera** located at the space origin and pointed in the
+negative direction of the Z-axis. In the current pipeline, it is assumed that
+the input camera frames are observed by exactly this virtual camera and
+therefore its parameters are later used to convert the screen landmark
+coordinates back into the Metric 3D space. The *virtual camera parameters* can
+be set freely, however for better results it is advised to set them as close to
+the *real physical camera parameters* as possible.
+
+![face_geometry_metric_3d_space.gif](../images/face_geometry_metric_3d_space.gif) |
+:----------------------------------------------------------------------------: |
+*Fig 3. A visualization of multiple key elements in the Metric 3D space.*      |
+
+#### Canonical Face Model
+
+The **Canonical Face Model** is a static 3D model of a human face, which follows
+the 468 3D face landmark topology of the
+[Face Landmark Model](#face-landmark-model). The model bears two important
+functions:
+
+-   **Defines metric units**: the scale of the canonical face model defines the
+    metric units of the Metric 3D space. A metric unit used by the
+    [default canonical face model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx)
+    is a centimeter;
+-   **Bridges static and runtime spaces**: the face pose transformation matrix
+    is - in fact - a linear map from the canonical face model into the runtime
+    face landmark set estimated on each frame. This way, virtual 3D assets
+    modeled around the canonical face model can be aligned with a tracked face
+    by applying the face pose transformation matrix to them.
+
+### Components
+
+#### Geometry Pipeline
+
+The **Geometry Pipeline** is a key component, which is responsible for
+estimating face geometry objects within the Metric 3D space. On each frame, the
+following steps are executed in the given order:
+
+-   Face landmark screen coordinates are converted into the Metric 3D space
+    coordinates;
+-   Face pose transformation matrix is estimated as a rigid linear mapping from
+    the canonical face metric landmark set into the runtime face metric landmark
+    set in a way that minimizes a difference between the two;
+-   A face mesh is created using the runtime face metric landmarks as the vertex
+    positions (XYZ), while both the vertex texture coordinates (UV) and the
+    triangular topology are inherited from the canonical face model.
+
+The geometry pipeline is implemented as a MediaPipe
+[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc).
+For your convenience, the face geometry pipeline calculator is bundled together
+with the face landmark module into a unified MediaPipe
+[subgraph](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_front_gpu.pbtxt).
+The face geometry format is defined as a Protocol Buffer
+[message](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto).
+
+#### Effect Renderer
+
+The **Effect Renderer** is a component, which serves as a working example of a
+face effect renderer. It targets the *OpenGL ES 2.0* API to enable a real-time
+performance on mobile devices and supports the following rendering modes:
+
+-   **3D object rendering mode**: a virtual object is aligned with a detected
+    face to emulate an object attached to the face (example: glasses);
+-   **Face mesh rendering mode**: a texture is stretched on top of the face mesh
+    surface to emulate a face painting technique.
+
+In both rendering modes, the face mesh is first rendered as an occluder straight
+into the depth buffer. This step helps to create a more believable effect via
+hiding invisible elements behind the face surface.
+
+The effect renderer is implemented as a MediaPipe
+[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc).
+
+| ![face_geometry_renderer.gif](../images/face_geometry_renderer.gif)     |
+| :---------------------------------------------------------------------: |
+| *Fig 4. An example of face effects rendered by the Face Geometry Effect Renderer.* |
 
 ## Example Apps
 
@@ -111,7 +217,12 @@ Note: To visualize a graph, copy the graph and paste it into
 to visualize its associated subgraphs, please see
 [visualizer documentation](../tools/visualizer.md).
 
-### Mobile
+### Face Landmark Example
+
+Face landmark example showcases real-time, cross-platform face landmark
+detection. For visual reference, please refer to *Fig. 2*.
+
+#### Mobile
 
 *   Graph:
     [`mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt)
@@ -127,7 +238,7 @@ it, for Android modify `NUM_FACES` in
 and for iOS modify `kNumFaces` in
 [FaceMeshGpuViewController.mm](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm).
 
-### Desktop
+#### Desktop
 
 *   Running on CPU
     *   Graph:
@@ -143,18 +254,35 @@ and for iOS modify `kNumFaces` in
 Tip: Maximum number of faces to detect/process is set to 1 by default. To change
 it, in the graph file modify the option of `ConstantSidePacketCalculator`.
 
+### Face Effect Example
+
+Face effect example showcases real-time mobile face effect application use case
+for the Face Mesh solution. To enable a better user experience, this example
+only works for a single face. For visual reference, please refer to *Fig. 4*.
+
+#### Mobile
+
+*   Graph:
+    [`mediapipe/graphs/face_effect/face_effect_gpu.pbtxt`](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/face_effect/face_effect_gpu.pbtxt)
+*   Android target:
+    [(or download prebuilt ARM64 APK)](https://drive.google.com/file/d/1ccnaDnffEuIXriBZr2SK_Eu4FpO7K44s)
+    [`mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect`](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD)
+*   iOS target:
+    [`mediapipe/examples/ios/faceeffect`](http:/mediapipe/examples/ios/faceeffect/BUILD)
+
 ## Resources
 
 *   Google AI Blog:
     [Real-Time AR Self-Expression with Machine Learning](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
 *   TensorFlow Blog:
     [Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
+*   Google Developers Blog:
+    [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog)
 *   Paper:
     [Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
     ([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
-*   Face detection model:
+*   Canonical face model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
+    [FBX](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.fbx),
-*   Face landmark model:
+    [OBJ](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model.obj),
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
+    [UV visualization](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png)
-    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
+*   [Models and model cards](./models.md#face_mesh)
-*   [Model card](https://mediapipe.page.link/facemesh-mc)

docs/solutions/hair_segmentation.md

@@ -54,5 +54,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
     [Real-time Hair segmentation and recoloring on Mobile GPUs](https://arxiv.org/abs/1907.06740)
     ([presentation](https://drive.google.com/file/d/1C8WYlWdDRNtU1_pYBvkkG5Z5wqYqf0yj/view))
     ([supplementary video](https://drive.google.com/file/d/1LPtM99Ch2ogyXYbDNpEqnUfhFq0TfLuf/view))
-*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
+*   [Models and model cards](./models.md#hair_segmentation)
-*   [Model card](https://mediapipe.page.link/hairsegmentation-mc)

docs/solutions/hands.md

@@ -226,10 +226,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [MediaPipe Hands: On-device Real-time Hand Tracking](https://arxiv.org/abs/2006.10214)
     ([presentation](https://www.youtube.com/watch?v=I-UOrvxxXEk))
-*   Palm detection model:
+*   [Models and model cards](./models.md#hands)
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
-*   Hand landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-*   [Model card](https://mediapipe.page.link/handmc)

docs/solutions/instant_motion_tracking.md

@@ -115,7 +115,7 @@ MediaPipe examples.
 ## Resources
 
 *   Google Developers Blog:
-  [Instant Motion Tracking With MediaPipe](https://mediapipe.page.link/instant-motion-tracking-blog)
+    [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
 *   Google AI Blog:
     [The Instant Motion Tracking Behind Motion Stills AR](https://ai.googleblog.com/2018/02/the-instant-motion-tracking-behind.html)
 *   Paper:

docs/solutions/iris.md

@@ -199,11 +199,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
     ([presentation](https://youtu.be/cIhXkiiapQI))
-*   Face detection model:
+*   [Models and model cards](./models.md#iris)
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_front.tflite)
-*   Face landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
-    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-*   Iris landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
-*   [Model card](https://mediapipe.page.link/iris-mc)

docs/solutions/knift.md

@@ -139,7 +139,4 @@ to run regular TFLite inference.
 
 *   Google Developers Blog:
     [MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
-*   [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
+*   [Models and model cards](./models.md#knift)
-*   [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
-*   [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
-*   [Model card](https://mediapipe.page.link/knift-mc)

docs/solutions/models.md

@@ -0,0 +1,77 @@
+---
+layout: default
+title: Models and Model Cards
+parent: Solutions
+nav_order: 30
+---
+
+# Models and Model Cards
+{: .no_toc }
+
+1. TOC
+{:toc}
+---
+
+### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
+
+*   Face detection model for front-facing/selfie camera:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite),
+    [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite)
+*   Face detection model for back-facing camera:
+    [TFLite model ](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
+*   [Model card](https://mediapipe.page.link/blazeface-mc)
+
+### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
+
+*   Face landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
+*   [Model card](https://mediapipe.page.link/facemesh-mc)
+
+### [Iris](https://google.github.io/mediapipe/solutions/iris)
+
+*   Iris landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
+*   [Model card](https://mediapipe.page.link/iris-mc)
+
+### [Hands](https://google.github.io/mediapipe/solutions/hands)
+
+*   Palm detection model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
+*   Hand landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite),
+    [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
+*   [Model card](https://mediapipe.page.link/handmc)
+
+### [Pose](https://google.github.io/mediapipe/solutions/pose)
+
+*   Pose detection model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
+*   Upper-body pose landmark model:
+    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
+*   [Model card](https://mediapipe.page.link/blazepose-mc)
+
+### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
+
+*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
+*   [Model card](https://mediapipe.page.link/hairsegmentation-mc)
+
+### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)
+
+*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
+*   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
+*   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
+*   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
+
+### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
+
+*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
+*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
+
+### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
+
+*   [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
+*   [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
+*   [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
+*   [Model card](https://mediapipe.page.link/knift-mc)

docs/solutions/object_detection.md

@@ -144,7 +144,4 @@ to cross-compile and run MediaPipe examples on the
 
 ## Resources
 
-*   [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
+*   [Models and model cards](./models.md#object_detection)
-*   [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
-*   [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
-*   [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)

docs/solutions/objectron.md

@@ -191,5 +191,4 @@ to visualize its associated subgraphs, please see
 *   Paper:
     [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8)
     ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0))
-*   [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
+*   [Models and model cards](./models.md#objectron)
-*   [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)

docs/solutions/pose.md

@@ -88,10 +88,12 @@ hip midpoints.
 ### Pose Landmark Model (BlazePose Tracker)
 
 The landmark model currently included in MediaPipe Pose predicts the location of
-25 upper-body landmarks (see figure below), with three degrees of freedom each
+25 upper-body landmarks (see figure below), each with `(x, y, z, visibility)`,
-(x, y location and visibility), plus two virtual alignment keypoints. It shares
+plus two virtual alignment keypoints. Note that the `z` value should be
-the same architecture as the full-body version that predicts 33 landmarks,
+discarded as the model is currently not fully trained to predict depth, but this
-described in more detail in the
+is something we have on the roadmap. The model shares the same architecture as
+the full-body version that predicts 33 landmarks, described in more detail in
+the
 [BlazePose Google AI Blog](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
 and in this [paper](https://arxiv.org/abs/2006.10204).
 
@@ -189,8 +191,4 @@ Please refer to [these instructions](../index.md#mediapipe-on-the-web).
 *   Paper:
     [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
     ([presentation](https://youtu.be/YPpUOTRn5tA))
-*   Pose detection model:
+*   [Models and model cards](./models.md#pose)
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
-*   Upper-body pose landmark model:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
-*   [Model card](https://mediapipe.page.link/blazepose-mc)

docs/tools/tracing_and_profiling.md

@@ -71,6 +71,9 @@ MediaPipe will emit data into a pre-specified directory:
 
     You can open the Download Container. Logs will be located in `application
     container/.xcappdata/AppData/Documents/`
+    If XCode shows empty content for the downloaded container file, you can
+    right click and select 'Show Package Contents' in Finder. Logs
+    will be located in 'AppData/Documents/'
 
     ![Windows Download Container](../images/visualizer/ios_download_container.png)
 

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -10,6 +10,7 @@
     "mediapipe/examples/ios/helloworld/BUILD",
     "mediapipe/examples/ios/facedetectioncpu/BUILD",
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
+    "mediapipe/examples/ios/faceeffect/BUILD",
     "mediapipe/examples/ios/facemeshgpu/BUILD",
     "mediapipe/examples/ios/handdetectiongpu/BUILD",
     "mediapipe/examples/ios/handtrackinggpu/BUILD",
@@ -23,6 +24,7 @@
     "//mediapipe/examples/ios/helloworld:HelloWorldApp",
     "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
+    "//mediapipe/examples/ios/faceeffect:FaceEffectApp",
     "//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp",
     "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp",
     "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp",
@@ -90,6 +92,8 @@
     "mediapipe/examples/ios/helloworld",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/faceeffect",
+    "mediapipe/examples/ios/faceeffect/Base.lproj",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",
     "mediapipe/examples/ios/iristrackinggpu",
@@ -110,6 +114,7 @@
     "mediapipe/graphs",
     "mediapipe/graphs/edge_detection",
     "mediapipe/graphs/face_detection",
+    "mediapipe/graphs/face_geometry",
     "mediapipe/graphs/hand_tracking",
     "mediapipe/graphs/object_detection",
     "mediapipe/graphs/pose_tracking",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -13,6 +13,7 @@
     "mediapipe/examples/ios",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
+    "mediapipe/examples/ios/faceeffect",
     "mediapipe/examples/ios/facemeshgpu",
     "mediapipe/examples/ios/handdetectiongpu",
     "mediapipe/examples/ios/handtrackinggpu",

mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -50,6 +50,9 @@ REGISTER_CALCULATOR(ConcatenateInt32VectorCalculator);
 typedef ConcatenateVectorCalculator<uint64> ConcatenateUInt64VectorCalculator;
 REGISTER_CALCULATOR(ConcatenateUInt64VectorCalculator);
 
+typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateBoolVectorCalculator);
+
 // Example config:
 // node {
 //   calculator: "ConcatenateTfLiteTensorVectorCalculator"

mediapipe/calculators/core/mux_calculator_test.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <memory>
+
 #include "mediapipe/calculators/core/split_vector_calculator.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
@@ -233,5 +235,71 @@ TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) {
            kOutputName, output_fn);
   EXPECT_EQ(output, input_packets);
 }
+
+constexpr char kDualInputGraphConfig[] = R"proto(
+  input_stream: "input_0"
+  input_stream: "input_1"
+  input_stream: "input_select"
+  output_stream: "test_output"
+  node {
+    calculator: "MuxCalculator"
+    input_stream: "INPUT:0:input_0"
+    input_stream: "INPUT:1:input_1"
+    input_stream: "SELECT:input_select"
+    output_stream: "OUTPUT:test_output"
+  }
+)proto";
+
+TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) {
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          kDualInputGraphConfig);
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  std::shared_ptr<int> output;
+  MP_ASSERT_OK(
+      graph.ObserveOutputStream("test_output", [&output](const Packet& p) {
+        output = p.Get<std::shared_ptr<int>>();
+        return ::mediapipe::OkStatus();
+      }));
+
+  MP_ASSERT_OK(graph.StartRun({}));
+
+  auto one = std::make_shared<int>(1);
+  auto two = std::make_shared<int>(2);
+  auto three = std::make_shared<int>(3);
+  std::weak_ptr<int> one_weak = one;
+  std::weak_ptr<int> two_weak = two;
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_0",
+      MakePacket<std::shared_ptr<int>>(std::move(one)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_1",
+      MakePacket<std::shared_ptr<int>>(std::move(two)).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_1",
+      MakePacket<std::shared_ptr<int>>(std::move(three)).At(Timestamp(1))));
+  EXPECT_EQ(one, nullptr);
+  EXPECT_EQ(two, nullptr);
+  EXPECT_EQ(three, nullptr);
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_select", MakePacket<int>(0).At(Timestamp(0))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(*output, 1);
+  EXPECT_NE(one_weak.lock(), nullptr);
+  EXPECT_EQ(two_weak.lock(), nullptr);
+
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_select", MakePacket<int>(1).At(Timestamp(1))));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(*output, 3);
+
+  MP_ASSERT_OK(graph.CloseAllInputStreams());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/calculators/util/BUILD

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
 
 licenses(["notice"])
 
@@ -945,6 +945,31 @@ cc_library(
     alwayslink = 1,
 )
 
+mediapipe_proto_library(
+    name = "landmarks_smoothing_calculator_proto",
+    srcs = ["landmarks_smoothing_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+cc_library(
+    name = "landmarks_smoothing_calculator",
+    srcs = ["landmarks_smoothing_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/util/filtering:relative_velocity_filter",
+        "@com_google_absl//absl/algorithm:container",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "landmarks_to_floats_calculator",
     srcs = ["landmarks_to_floats_calculator.cc"],
@@ -1103,6 +1128,7 @@ cc_library(
         "//mediapipe/framework/formats:classification_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@com_google_absl//absl/strings",

mediapipe/calculators/util/filter_collection_calculator.cc

@@ -20,9 +20,14 @@
 #include "mediapipe/framework/formats/classification.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/integral_types.h"
 
 namespace mediapipe {
 
+typedef FilterCollectionCalculator<std::vector<uint64>>
+    FilterUInt64CollectionCalculator;
+REGISTER_CALCULATOR(FilterUInt64CollectionCalculator);
+
 typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
     FilterNormalizedRectCollectionCalculator;
 REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -13,12 +13,12 @@
 // limitations under the License.
 
 #include "absl/algorithm/container.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
-#include "mediapipe/graphs/pose_tracking/calculators/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
-#include "mediapipe/graphs/pose_tracking/calculators/relative_velocity_filter.h"
 
 namespace mediapipe {
 
@@ -38,17 +38,17 @@ using ::mediapipe::RelativeVelocityFilter;
 // with sides parallel to axis.
 float GetObjectScale(const NormalizedLandmarkList& landmarks, int image_width,
                      int image_height) {
-  const auto& [lm_min_x, lm_max_x] = absl::c_minmax_element(
+  const auto& lm_minmax_x = absl::c_minmax_element(
       landmarks.landmark(),
       [](const auto& a, const auto& b) { return a.x() < b.x(); });
-  const float x_min = lm_min_x->x();
+  const float x_min = lm_minmax_x.first->x();
-  const float x_max = lm_max_x->x();
+  const float x_max = lm_minmax_x.second->x();
 
-  const auto& [lm_min_y, lm_max_y] = absl::c_minmax_element(
+  const auto& lm_minmax_y = absl::c_minmax_element(
       landmarks.landmark(),
       [](const auto& a, const auto& b) { return a.y() < b.y(); });
-  const float y_min = lm_min_y->y();
+  const float y_min = lm_minmax_y.first->y();
-  const float y_max = lm_max_y->y();
+  const float y_max = lm_minmax_y.second->y();
 
   const float object_width = (x_max - x_min) * image_width;
   const float object_height = (y_max - y_min) * image_height;
@@ -191,8 +191,8 @@ class VelocityFilter : public LandmarksFilter {
 //     input_stream: "NORM_LANDMARKS:pose_landmarks"
 //     input_stream: "IMAGE_SIZE:image_size"
 //     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
-//     node_options: {
+//     options: {
-//       [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
 //         velocity_filter: {
 //           window_size: 5
 //           velocity_scale: 10.0

mediapipe/calculators/util/landmarks_smoothing_calculator.proto

@@ -16,7 +16,7 @@ syntax = "proto2";
 
 package mediapipe;
 
-import "mediapipe/framework/calculator.proto";
+import "mediapipe/framework/calculator_options.proto";
 
 message LandmarksSmoothingCalculatorOptions {
   extend CalculatorOptions {

mediapipe/calculators/util/landmarks_to_render_data_calculator.cc

@@ -103,7 +103,7 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
+    if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
                                ld1.visibility() < visibility_threshold)) {
       continue;
     }
@@ -141,7 +141,7 @@ void AddConnections(const LandmarkListType& landmarks,
   for (int i = 0; i < landmark_connections.size(); i += 2) {
     const auto& ld0 = landmarks.landmark(landmark_connections[i]);
     const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]);
-    if (visibility_threshold && (ld0.visibility() < visibility_threshold ||
+    if (utilize_visibility && (ld0.visibility() < visibility_threshold ||
                                ld1.visibility() < visibility_threshold)) {
       continue;
     }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD

@@ -0,0 +1,67 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+android_binary(
+    name = "faceeffect",
+    srcs = glob(["*.java"]),
+    assets = [
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    assets_dir = "",
+    manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",
+    manifest_values = {
+        "applicationId": "com.google.mediapipe.apps.faceeffect",
+        "appName": "Face Effect",
+        "mainActivity": ".MainActivity",
+        "cameraFacingFront": "True",
+        "binaryGraphName": "face_effect_gpu.binarypb",
+        "inputVideoStreamName": "input_video",
+        "outputVideoStreamName": "output_video",
+        "flipFramesVertically": "True",
+    },
+    multidex = "native",
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib",
+        "//mediapipe/framework/formats:matrix_data_java_proto_lite",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java

@@ -0,0 +1,176 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.faceeffect;
+
+import android.graphics.Color;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.GestureDetector;
+import android.view.Gravity;
+import android.view.MotionEvent;
+import android.view.View;
+import android.view.ViewGroup;
+import android.view.ViewGroup.LayoutParams;
+import android.widget.RelativeLayout;
+import android.widget.TextView;
+import com.google.mediapipe.framework.Packet;
+import com.google.mediapipe.framework.PacketGetter;
+import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry;
+import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData;
+import java.util.List;
+
+/** Main activity of MediaPipe face mesh app. */
+public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity {
+  private static final String TAG = "MainActivity";
+
+  // Stream names.
+  private static final String IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME =
+      "is_facepaint_effect_selected";
+  private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry";
+
+  private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!";
+
+  private static final int MATRIX_TRANSLATION_Z_INDEX = 14;
+
+  private final Object isFacepaintEffectSelectedLock = new Object();
+  private boolean isFacepaintEffectSelected;
+
+  private View effectSwitchingHintView;
+  private GestureDetector tapGestureDetector;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+
+    // Add an effect switching hint view to the preview layout.
+    effectSwitchingHintView = createEffectSwitchingHintView();
+    effectSwitchingHintView.setVisibility(View.INVISIBLE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(effectSwitchingHintView);
+
+    // By default, render the glasses effect.
+    isFacepaintEffectSelected = false;
+
+    // This callback demonstrates how the output face geometry packet can be obtained and used
+    // in an Android app. As an example, the Z-translation component of the face pose transform
+    // matrix is logged for each face being equal to the approximate distance away from the camera
+    // in centimeters.
+    processor.addPacketCallback(
+        OUTPUT_FACE_GEOMETRY_STREAM_NAME,
+        (packet) -> {
+          effectSwitchingHintView.post(
+              new Runnable() {
+                @Override
+                public void run() {
+                  effectSwitchingHintView.setVisibility(View.VISIBLE);
+                }
+              });
+
+          Log.d(TAG, "Received a multi face geometry packet.");
+          List<FaceGeometry> multiFaceGeometry =
+              PacketGetter.getProtoVector(packet, FaceGeometry.parser());
+
+          StringBuilder approxDistanceAwayFromCameraLogMessage = new StringBuilder();
+          for (FaceGeometry faceGeometry : multiFaceGeometry) {
+            if (approxDistanceAwayFromCameraLogMessage.length() > 0) {
+              approxDistanceAwayFromCameraLogMessage.append(' ');
+            }
+            MatrixData poseTransformMatrix = faceGeometry.getPoseTransformMatrix();
+            approxDistanceAwayFromCameraLogMessage.append(
+                -poseTransformMatrix.getPackedData(MATRIX_TRANSLATION_Z_INDEX));
+          }
+
+          Log.d(
+              TAG,
+              "[TS:"
+                  + packet.getTimestamp()
+                  + "] size = "
+                  + multiFaceGeometry.size()
+                  + "; approx. distance away from camera in cm for faces = ["
+                  + approxDistanceAwayFromCameraLogMessage
+                  + "]");
+        });
+
+    // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+    // packet to indicate which effect should be rendered on this frame.
+    processor.setOnWillAddFrameListener(
+        (timestamp) -> {
+          Packet isFacepaintEffectSelectedPacket = null;
+          try {
+            synchronized (isFacepaintEffectSelectedLock) {
+              isFacepaintEffectSelectedPacket =
+                  processor.getPacketCreator().createBool(isFacepaintEffectSelected);
+            }
+
+            processor
+                .getGraph()
+                .addPacketToInputStream(
+                    IS_FACEPAINT_EFFECT_SELECTED_INPUT_STREAM_NAME,
+                    isFacepaintEffectSelectedPacket,
+                    timestamp);
+          } catch (RuntimeException e) {
+            Log.e(
+                TAG,
+                "Exception while adding packet to input stream while switching effects: " + e);
+          } finally {
+            if (isFacepaintEffectSelectedPacket != null) {
+              isFacepaintEffectSelectedPacket.release();
+            }
+          }
+        });
+
+    // We use the tap gesture detector to switch between face effects. This allows users to try
+    // multiple pre-bundled face effects without a need to recompile the app.
+    tapGestureDetector =
+        new GestureDetector(
+            this,
+            new GestureDetector.SimpleOnGestureListener() {
+              @Override
+              public void onLongPress(MotionEvent event) {
+                switchEffect();
+              }
+
+              @Override
+              public boolean onSingleTapUp(MotionEvent event) {
+                switchEffect();
+                return true;
+              }
+
+              private void switchEffect() {
+                synchronized (isFacepaintEffectSelectedLock) {
+                  isFacepaintEffectSelected = !isFacepaintEffectSelected;
+                }
+              }
+            });
+  }
+
+  @Override
+  public boolean onTouchEvent(MotionEvent event) {
+    return tapGestureDetector.onTouchEvent(event);
+  }
+
+  private View createEffectSwitchingHintView() {
+    TextView effectSwitchingHintView = new TextView(getApplicationContext());
+    effectSwitchingHintView.setLayoutParams(
+        new RelativeLayout.LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.FILL_PARENT));
+    effectSwitchingHintView.setText(EFFECT_SWITCHING_HINT_TEXT);
+    effectSwitchingHintView.setGravity(Gravity.CENTER_HORIZONTAL | Gravity.BOTTOM);
+    effectSwitchingHintView.setPadding(0, 0, 0, 480);
+    effectSwitchingHintView.setTextColor(Color.parseColor("#ffffff"));
+    effectSwitchingHintView.setTextSize((float) 24);
+
+    return effectSwitchingHintView;
+  }
+}

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc

@@ -35,7 +35,7 @@ constexpr char kDetectedBorders[] = "BORDERS";
 constexpr char kCropRect[] = "CROP_RECT";
 // Field-of-view (degrees) of the camera's x-axis (width).
 // TODO: Parameterize FOV based on camera specs.
-constexpr float kWidthFieldOfView = 60;
+constexpr float kFieldOfView = 60;
 
 namespace mediapipe {
 namespace autoflip {
@@ -256,13 +256,13 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
   if (!initialized_) {
     path_solver_height_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_zoom(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
     path_solver_width_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_pan(), 0, frame_width_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_width_) / kFieldOfView);
     path_solver_offset_ = std::make_unique<KinematicPathSolver>(
         options_.kinematic_options_tilt(), 0, frame_height_,
-        static_cast<float>(frame_width_) / kWidthFieldOfView);
+        static_cast<float>(frame_height_) / kFieldOfView);
     max_frame_value_ = 1.0;
     target_aspect_ = frame_width_ / static_cast<float>(frame_height_);
     // If target size is set and wider than input aspect, make sure to always
@@ -339,15 +339,24 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
     offset_y = last_measured_y_offset_;
   }
 
-  // Compute smoothed camera paths.
+  // Compute smoothed zoom camera path.
   MP_RETURN_IF_ERROR(path_solver_height_->AddObservation(
       height, cc->InputTimestamp().Microseconds()));
+  int path_height;
+  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
+  int path_width = path_height * target_aspect_;
+
+  // Update pixel-per-degree value for pan/tilt.
+  MP_RETURN_IF_ERROR(path_solver_width_->UpdatePixelsPerDegree(
+      static_cast<float>(path_width) / kFieldOfView));
+  MP_RETURN_IF_ERROR(path_solver_offset_->UpdatePixelsPerDegree(
+      static_cast<float>(path_height) / kFieldOfView));
+
+  // Compute smoothed pan/tilt paths.
   MP_RETURN_IF_ERROR(path_solver_width_->AddObservation(
       offset_x, cc->InputTimestamp().Microseconds()));
   MP_RETURN_IF_ERROR(path_solver_offset_->AddObservation(
       offset_y, cc->InputTimestamp().Microseconds()));
-  int path_height;
-  MP_RETURN_IF_ERROR(path_solver_height_->GetState(&path_height));
   int path_offset_x;
   MP_RETURN_IF_ERROR(path_solver_width_->GetState(&path_offset_x));
   int path_offset_y;
@@ -359,7 +368,7 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
   } else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
     path_offset_y = frame_height_ - ceil(path_height / 2.0);
   }
-  int path_width = path_height * target_aspect_;
+
   if (path_offset_x - ceil(path_width / 2.0) < 0) {
     path_offset_x = ceil(path_width / 2.0);
   } else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc

@@ -174,15 +174,15 @@ TEST(ContentZoomingCalculatorTest, PanConfig) {
       ContentZoomingCalculatorOptions::ext);
   options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_pan()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);
   AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
   AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
   MP_ASSERT_OK(runner->Run());
   CheckCropRect(450, 550, 111, 111, 0,
                 runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(488, 550, 111, 111, 1,
+  CheckCropRect(483, 550, 111, 111, 1,
                 runner->Outputs().Tag("CROP_RECT").packets);
 }
 
@@ -190,17 +190,17 @@ TEST(ContentZoomingCalculatorTest, TiltConfig) {
   auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
   auto* options = config.mutable_options()->MutableExtension(
       ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
   options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2);
-  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);
   AddDetection(cv::Rect_<float>(.4, .5, .1, .1), 0, runner.get());
   AddDetection(cv::Rect_<float>(.45, .55, .15, .15), 1000000, runner.get());
   MP_ASSERT_OK(runner->Run());
   CheckCropRect(450, 550, 111, 111, 0,
                 runner->Outputs().Tag("CROP_RECT").packets);
-  CheckCropRect(450, 588, 111, 111, 1,
+  CheckCropRect(450, 583, 111, 111, 1,
                 runner->Outputs().Tag("CROP_RECT").packets);
 }
 
@@ -208,8 +208,8 @@ TEST(ContentZoomingCalculatorTest, ZoomConfig) {
   auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
   auto* options = config.mutable_options()->MutableExtension(
       ContentZoomingCalculatorOptions::ext);
-  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0);
-  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(5.0);
+  options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0);
   options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0);
   options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2);
   auto runner = ::absl::make_unique<CalculatorRunner>(config);

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc

@@ -87,5 +87,13 @@ namespace autoflip {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status KinematicPathSolver::UpdatePixelsPerDegree(
+    const float pixels_per_degree) {
+  RET_CHECK_GT(pixels_per_degree_, 0)
+      << "pixels_per_degree must be larger than 0.";
+  pixels_per_degree_ = pixels_per_degree;
+  return ::mediapipe::OkStatus();
+}
+
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h

@@ -46,6 +46,8 @@ class KinematicPathSolver {
   ::mediapipe::Status UpdatePrediction(const int64 time_us);
   // Get the state at a time.
   ::mediapipe::Status GetState(int* position);
+  // Update PixelPerDegree value.
+  ::mediapipe::Status UpdatePixelsPerDegree(const float pixels_per_degree);
 
  private:
   // Tuning options.

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc

@@ -207,6 +207,28 @@ TEST(KinematicPathSolverTest, PassMaxVelocity) {
   EXPECT_EQ(state, 600);
 }
 
+TEST(KinematicPathSolverTest, PassDegPerPxChange) {
+  KinematicOptions options;
+  // Set min motion to 2deg
+  options.set_min_motion_to_reframe(2.0);
+  options.set_update_rate(1);
+  options.set_max_velocity(1000);
+  // Set degrees / pixel to 16.6
+  KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView);
+  int state;
+  MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0));
+  // Move target by 20px / 16.6 = 1.2deg
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to not move.
+  EXPECT_EQ(state, 500);
+  MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView));
+  MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to move.
+  EXPECT_EQ(state, 516);
+}
+
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt

@@ -1,5 +1,5 @@
 # MediaPipe graph that performs face detection with TensorFlow Lite on CPU.  Model paths setup for web use.
-# TODO: parameterize input paths to support desktop use.
+# TODO: parameterize input paths to support desktop use, for web only.
 input_stream: "VIDEO:input_video"
 output_stream: "DETECTIONS:output_detections"
 
@@ -37,7 +37,7 @@ node {
   output_stream: "TENSORS:detection_tensors"
   options: {
     [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
-      model_path: "mediapipe/models/face_detection_front.tflite"
+      model_path: "face_detection_front.tflite"
     }
   }
 }
@@ -118,7 +118,7 @@ node {
   output_stream: "labeled_detections"
   options: {
     [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
-      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+      label_map_path: "face_detection_front_labelmap.txt"
     }
   }
 }

mediapipe/examples/ios/faceeffect/AppDelegate.h

@@ -0,0 +1,21 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

mediapipe/examples/ios/faceeffect/AppDelegate.m

@@ -0,0 +1,59 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end

mediapipe/examples/ios/faceeffect/BUILD

@@ -0,0 +1,92 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+load(
+    "//mediapipe/examples/ios:bundle_id.bzl",
+    "BUNDLE_ID_PREFIX",
+    "example_provisioning",
+)
+
+licenses(["notice"])
+
+MIN_IOS_VERSION = "10.0"
+
+alias(
+    name = "faceeffect",
+    actual = "FaceEffectApp",
+)
+
+ios_application(
+    name = "FaceEffectApp",
+    app_icons = ["//mediapipe/examples/ios/common:AppIcon"],
+    bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = example_provisioning(),
+    deps = [
+        ":FaceEffectAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "FaceEffectAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "FaceEffectViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "FaceEffectViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/modules/face_detection:face_detection_front.tflite",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:matrix_data_cc_proto",
+            "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+            "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        ],
+    }),
+)

mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="16096" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina5_5" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="16086"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="FaceEffectViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="736"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="414" height="716"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="76" y="283" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Tap to switch between effects!" textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="b2C-zd-Zba" userLabel="Effect Switching Label">
+                                        <rect key="frame" x="77" y="488" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_effectSwitchingHintLabel" destination="b2C-zd-Zba" id="Uvx-2n-l74"/>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/faceeffect/FaceEffectViewController.h

@@ -0,0 +1,19 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface FaceEffectViewController : UIViewController
+
+@end

mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm

@@ -0,0 +1,254 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "FaceEffectViewController.h"
+
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+#include <utility>
+
+#include "mediapipe/framework/formats/matrix_data.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+
+static NSString* const kGraphName = @"face_effect_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kIsFacepaintEffectSelectedInputStream = "is_facepaint_effect_selected";
+static const char* kOutputStream = "output_video";
+static const char* kMultiFaceGeometryStream = "multi_face_geometry";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+static const int kMatrixTranslationZIndex = 14;
+
+@interface FaceEffectViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* graph;
+
+@end
+
+@implementation FaceEffectViewController {
+  /// Handle tap gestures.
+  UITapGestureRecognizer* _tapGestureRecognizer;
+  BOOL _isFacepaintEffectSelected;
+
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Inform the user about how to switch between effects.
+  UILabel* _effectSwitchingHintLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.graph.delegate = nil;
+  [self.graph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.graph closeAllInputStreamsWithError:nil];
+  [self.graph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  [newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _effectSwitchingHintLabel.hidden = YES;
+  _tapGestureRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self
+                                                                  action:@selector(handleTap)];
+  [self.view addGestureRecognizer:_tapGestureRecognizer];
+
+  // By default, render the glasses effect.
+  _isFacepaintEffectSelected = NO;
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer insertSublayer:_renderer.layer atIndex:0];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  _renderer.mirrored = NO;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+  _cameraSource.videoMirrored = YES;
+
+  self.graph = [[self class] loadGraphFromResource:kGraphName];
+  self.graph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.graph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.graph.
+  NSError* error;
+  if (![self.graph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - UITapGestureRecognizer methods
+
+// We use the tap gesture recognizer to switch between face effects. This allows users to try
+// multiple pre-bundled face effects without a need to recompile the app.
+- (void)handleTap {
+  dispatch_async(_videoQueue, ^{
+    _isFacepaintEffectSelected = !_isFacepaintEffectSelected;
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      _effectSwitchingHintLabel.hidden = NO;
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+//
+// This callback demonstrates how the output face geometry packet can be obtained and used in an
+// iOS app. As an example, the Z-translation component of the face pose transform matrix is logged
+// for each face being equal to the approximate distance away from the camera in centimeters.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kMultiFaceGeometryStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face geometry", packet.Timestamp().Value());
+      return;
+    }
+
+    const auto& multiFaceGeometry =
+        packet.Get<std::vector<::mediapipe::face_geometry::FaceGeometry>>();
+    NSLog(@"[TS:%lld] Number of face instances with geometry: %lu ", packet.Timestamp().Value(),
+          multiFaceGeometry.size());
+    for (int faceIndex = 0; faceIndex < multiFaceGeometry.size(); ++faceIndex) {
+      const auto& faceGeometry = multiFaceGeometry[faceIndex];
+      NSLog(@"\tApprox. distance away from camera for face[%d]: %.6f cm", faceIndex,
+            -faceGeometry.pose_transform_matrix().packed_data(kMatrixTranslationZIndex));
+    }
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+
+  mediapipe::Timestamp graphTimestamp(static_cast<mediapipe::TimestampBaseType>(
+      mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp)));
+
+  mediapipe::Packet isFacepaintEffectSelectedPacket =
+      mediapipe::MakePacket<bool>(_isFacepaintEffectSelected).At(graphTimestamp);
+
+  [self.graph sendPixelBuffer:imageBuffer
+                   intoStream:kInputStream
+                   packetType:MPPPacketTypePixelBuffer
+                    timestamp:graphTimestamp];
+
+  // Alongside the input camera frame, we also send the `is_facepaint_effect_selected` boolean
+  // packet to indicate which effect should be rendered on this frame.
+  [self.graph movePacket:std::move(isFacepaintEffectSelectedPacket)
+              intoStream:kIsFacepaintEffectSelectedInputStream
+                   error:nil];
+}
+
+@end

mediapipe/examples/ios/faceeffect/Info.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>

mediapipe/examples/ios/faceeffect/main.m

@@ -0,0 +1,22 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char* argv[]) {
+  @autoreleasepool {
+    return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}

mediapipe/examples/python/upper_body_pose_tracker.py

@@ -150,7 +150,7 @@ class UpperBodyPoseTracker:
       success, input_frame = cap.read()
       if not success:
         break
-      input_frame = cv2.cvtColor(input_frame, cv2.COLOR_BGR2RGB)
+      input_frame = cv2.cvtColor(cv2.flip(input_frame, 1), cv2.COLOR_BGR2RGB)
       input_frame.flags.writeable = False
       _, output_frame = self._run_graph(input_frame)
       cv2.imshow('MediaPipe upper body pose tracker',

mediapipe/framework/BUILD

@@ -947,6 +947,7 @@ cc_library(
         ],
     }),
     visibility = [
+        "//mediapipe/calculators:__subpackages__",
         "//mediapipe/framework:__subpackages__",
         "//mediapipe/framework/port:__pkg__",
         "//mediapipe/util:__subpackages__",

mediapipe/framework/calculator_graph.cc

@@ -330,6 +330,9 @@ CalculatorGraph::~CalculatorGraph() {
 ::mediapipe::Status CalculatorGraph::InitializeDefaultExecutor(
     const ThreadPoolExecutorOptions* default_executor_options,
     bool use_application_thread) {
+#ifdef __EMSCRIPTEN__
+  use_application_thread = true;
+#endif  // __EMSCRIPTEN__
   // If specified, run synchronously on the calling thread.
   if (use_application_thread) {
     use_application_thread_ = true;

mediapipe/framework/deps/file_helpers.cc

@@ -27,7 +27,9 @@
 
 #include "mediapipe/framework/deps/canonical_errors.h"
 #include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/deps/status.h"
 #include "mediapipe/framework/deps/status_builder.h"
+#include "mediapipe/framework/deps/status_macros.h"
 
 namespace mediapipe {
 namespace file {
@@ -212,7 +214,7 @@ class DirectoryListing {
 ::mediapipe::Status Exists(absl::string_view file_name) {
   struct stat buffer;
   int status;
-  status = stat(file_name.data(), &buffer);
+  status = stat(std::string(file_name).c_str(), &buffer);
   if (status == 0) {
     return ::mediapipe::OkStatus();
   }
@@ -224,5 +226,30 @@ class DirectoryListing {
   }
 }
 
+#ifndef _WIN32
+int mkdir(std::string path) {
+  return ::mkdir(path.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+}
+#else
+int mkdir(std::string path) { return _mkdir(path.c_str()); }
+#endif
+
+::mediapipe::Status RecursivelyCreateDir(absl::string_view path) {
+  if (path.empty() || Exists(path).ok()) {
+    return mediapipe::OkStatus();
+  }
+  auto split_path = file::SplitPath(path);
+  MP_RETURN_IF_ERROR(RecursivelyCreateDir(split_path.first));
+  if (mkdir(std::string(path)) != 0) {
+    switch (errno) {
+      case EACCES:
+        return ::mediapipe::PermissionDeniedError("Insufficient permissions.");
+      default:
+        return ::mediapipe::UnavailableError("Failed to create directory.");
+    }
+  }
+  return mediapipe::OkStatus();
+}
+
 }  // namespace file
 }  // namespace mediapipe

mediapipe/framework/deps/file_helpers.h

@@ -36,6 +36,8 @@ namespace file {
 
 ::mediapipe::Status Exists(absl::string_view file_name);
 
+::mediapipe::Status RecursivelyCreateDir(absl::string_view path);
+
 }  // namespace file
 }  // namespace mediapipe
 

mediapipe/framework/deps/threadpool_pthread_impl.cc

@@ -47,7 +47,8 @@ class ThreadPool::WorkerThread {
 ThreadPool::WorkerThread::WorkerThread(ThreadPool* pool,
                                        const std::string& name_prefix)
     : pool_(pool), name_prefix_(name_prefix) {
-  pthread_create(&thread_, nullptr, ThreadBody, this);
+  int res = pthread_create(&thread_, nullptr, ThreadBody, this);
+  CHECK_EQ(res, 0) << "pthread_create failed";
 }
 
 ThreadPool::WorkerThread::~WorkerThread() {}
@@ -59,9 +60,9 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
   int nice_priority_level =
       thread->pool_->thread_options().nice_priority_level();
   const std::set<int> selected_cpus = thread->pool_->thread_options().cpu_set();
+#if defined(__linux__)
   const std::string name =
       internal::CreateThreadName(thread->name_prefix_, syscall(SYS_gettid));
-#if defined(__linux__)
   if (nice_priority_level != 0) {
     if (nice(nice_priority_level) != -1 || errno == 0) {
       VLOG(1) << "Changed the nice priority level by " << nice_priority_level;
@@ -94,16 +95,19 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) {
                << "Failed to set name for thread: " << name;
   }
 #else
+  const std::string name = internal::CreateThreadName(thread->name_prefix_, 0);
   if (nice_priority_level != 0 || !selected_cpus.empty()) {
     LOG(ERROR) << "Thread priority and processor affinity feature aren't "
                   "supported on the current platform.";
   }
+#if __APPLE__
   int error = pthread_setname_np(name.c_str());
   if (error != 0) {
     LOG(ERROR) << "Error : " << strerror(error) << std::endl
                << "Failed to set name for thread: " << name;
   }
-#endif
+#endif  // __APPLE__
+#endif  // __linux__
   thread->pool_->RunWorker();
   return nullptr;
 }
@@ -178,6 +182,12 @@ const ThreadOptions& ThreadPool::thread_options() const {
 
 namespace internal {
 
+// TODO: revise this:
+// - thread_id is not portable
+// - the 16-byte limit is Linux-specific
+// - the std::thread implementation has a copy of this but doesn't use it
+// - why do we even need the thread id in the name? any thread list should show
+//   the id too.
 std::string CreateThreadName(const std::string& prefix, int thread_id) {
   std::string name = absl::StrCat(prefix, "/", thread_id);
   // 16 is the limit allowed by `pthread_setname_np`, including

mediapipe/framework/encode_binary_proto.bzl

@@ -1,3 +1,17 @@
+# Copyright 2019-2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """A rule for encoding a text format protocol buffer into binary.
 
 Example usage:

mediapipe/framework/formats/BUILD

@@ -227,3 +227,28 @@ filegroup(
     srcs = glob(["*.proto"]),
     visibility = ["//mediapipe:__subpackages__"],
 )
+
+cc_library(
+    name = "image_frame_pool",
+    srcs = ["image_frame_pool.cc"],
+    hdrs = ["image_frame_pool.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":image_frame",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
+cc_test(
+    name = "image_frame_pool_test",
+    size = "small",
+    srcs = ["image_frame_pool_test.cc"],
+    tags = ["linux"],
+    deps = [
+        ":image_frame_pool",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)

mediapipe/framework/formats/image_frame_pool.cc

@@ -0,0 +1,88 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/image_frame_pool.h"
+
+#include "absl/synchronization/mutex.h"
+
+namespace mediapipe {
+
+ImageFramePool::ImageFramePool(int width, int height,
+                               ImageFormat::Format format, int keep_count)
+    : width_(width),
+      height_(height),
+      format_(format),
+      keep_count_(keep_count) {}
+
+ImageFrameSharedPtr ImageFramePool::GetBuffer() {
+  std::unique_ptr<ImageFrame> buffer;
+
+  {
+    absl::MutexLock lock(&mutex_);
+    if (available_.empty()) {
+      // Fix alignment at 4 for best compatability with OpenGL.
+      buffer = std::make_unique<ImageFrame>(
+          format_, width_, height_, ImageFrame::kGlDefaultAlignmentBoundary);
+      if (!buffer) return nullptr;
+    } else {
+      buffer = std::move(available_.back());
+      available_.pop_back();
+    }
+
+    ++in_use_count_;
+  }
+
+  // Return a shared_ptr with a custom deleter that adds the buffer back
+  // to our available list.
+  std::weak_ptr<ImageFramePool> weak_pool(shared_from_this());
+  return std::shared_ptr<ImageFrame>(buffer.release(),
+                                     [weak_pool](ImageFrame* buf) {
+                                       auto pool = weak_pool.lock();
+                                       if (pool) {
+                                         pool->Return(buf);
+                                       } else {
+                                         delete buf;
+                                       }
+                                     });
+}
+
+std::pair<int, int> ImageFramePool::GetInUseAndAvailableCounts() {
+  absl::MutexLock lock(&mutex_);
+  return {in_use_count_, available_.size()};
+}
+
+void ImageFramePool::Return(ImageFrame* buf) {
+  std::vector<std::unique_ptr<ImageFrame>> trimmed;
+  {
+    absl::MutexLock lock(&mutex_);
+    --in_use_count_;
+    available_.emplace_back(buf);
+    TrimAvailable(&trimmed);
+  }
+  // The trimmed buffers will be released without holding the lock.
+}
+
+void ImageFramePool::TrimAvailable(
+    std::vector<std::unique_ptr<ImageFrame>>* trimmed) {
+  int keep = std::max(keep_count_ - in_use_count_, 0);
+  if (available_.size() > keep) {
+    auto trim_it = std::next(available_.begin(), keep);
+    if (trimmed) {
+      std::move(trim_it, available_.end(), std::back_inserter(*trimmed));
+    }
+    available_.erase(trim_it, available_.end());
+  }
+}
+
+}  // namespace mediapipe

mediapipe/framework/formats/image_frame_pool.h

@@ -0,0 +1,78 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Consider this file an implementation detail. None of this is part of the
+// public API.
+
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_
+
+#include <utility>
+#include <vector>
+
+#include "absl/synchronization/mutex.h"
+#include "mediapipe/framework/formats/image_frame.h"
+
+namespace mediapipe {
+
+using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
+
+class ImageFramePool : public std::enable_shared_from_this<ImageFramePool> {
+ public:
+  // Creates a pool. This pool will manage buffers of the specified dimensions,
+  // and will keep keep_count buffers around for reuse.
+  // We enforce creation as a shared_ptr so that we can use a weak reference in
+  // the buffers' deleters.
+  static std::shared_ptr<ImageFramePool> Create(int width, int height,
+                                                ImageFormat::Format format,
+                                                int keep_count) {
+    return std::shared_ptr<ImageFramePool>(
+        new ImageFramePool(width, height, format, keep_count));
+  }
+
+  // Obtains a buffers. May either be reused or created anew.
+  ImageFrameSharedPtr GetBuffer();
+
+  int width() const { return width_; }
+  int height() const { return height_; }
+  ImageFormat::Format format() const { return format_; }
+
+  // This method is meant for testing.
+  std::pair<int, int> GetInUseAndAvailableCounts();
+
+ private:
+  ImageFramePool(int width, int height, ImageFormat::Format format,
+                 int keep_count);
+
+  // Return a buffer to the pool.
+  void Return(ImageFrame* buf);
+
+  // If the total number of buffers is greater than keep_count, destroys any
+  // surplus buffers that are no longer in use.
+  void TrimAvailable(std::vector<std::unique_ptr<ImageFrame>>* trimmed)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+  const int width_;
+  const int height_;
+  const ImageFormat::Format format_;
+  const int keep_count_;
+
+  absl::Mutex mutex_;
+  int in_use_count_ ABSL_GUARDED_BY(mutex_) = 0;
+  std::vector<std::unique_ptr<ImageFrame>> available_ ABSL_GUARDED_BY(mutex_);
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_POOL_H_

mediapipe/framework/formats/image_frame_pool_test.cc

@@ -0,0 +1,103 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/formats/image_frame_pool.h"
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace {
+
+using Pair = std::pair<int, int>;
+
+constexpr int kWidth = 300;
+constexpr int kHeight = 200;
+constexpr ImageFormat::Format kFormat = ImageFormat::SRGBA;
+constexpr int kKeepCount = 2;
+
+class ImageFramePoolTest : public ::testing::Test {
+ protected:
+  ImageFramePoolTest() {
+    pool_ = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
+  }
+
+  void SetUp() override {}
+
+  std::shared_ptr<ImageFramePool> pool_;
+};
+
+TEST_F(ImageFramePoolTest, GetBuffer) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  auto buffer = pool_->GetBuffer();
+  EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
+  buffer = nullptr;
+  EXPECT_EQ(Pair(0, 1), pool_->GetInUseAndAvailableCounts());
+  buffer = pool_->GetBuffer();
+  EXPECT_EQ(Pair(1, 0), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST_F(ImageFramePoolTest, GetMoreBuffers) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  std::vector<ImageFrameSharedPtr> buffers;
+
+  // Create kKeepCount + 1 buffers
+  for (int i = 0; i <= kKeepCount; i++) {
+    buffers.emplace_back(pool_->GetBuffer());
+  }
+  EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete one
+  buffers.resize(kKeepCount);
+  EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete all
+  buffers.resize(0);
+  EXPECT_EQ(Pair(0, kKeepCount), pool_->GetInUseAndAvailableCounts());
+
+  // Create one more
+  buffers.emplace_back(pool_->GetBuffer());
+  EXPECT_EQ(Pair(1, kKeepCount - 1), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST_F(ImageFramePoolTest, DeleteNotLast) {
+  EXPECT_EQ(Pair(0, 0), pool_->GetInUseAndAvailableCounts());
+  std::vector<ImageFrameSharedPtr> buffers;
+
+  // Create kKeepCount + 1 buffers
+  for (int i = 0; i <= kKeepCount; i++) {
+    buffers.emplace_back(pool_->GetBuffer());
+  }
+  EXPECT_EQ(Pair(kKeepCount + 1, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete second
+  buffers.erase(buffers.begin() + 1);
+  EXPECT_EQ(Pair(kKeepCount, 0), pool_->GetInUseAndAvailableCounts());
+
+  // Delete first
+  buffers.erase(buffers.begin());
+  EXPECT_EQ(Pair(kKeepCount - 1, 1), pool_->GetInUseAndAvailableCounts());
+}
+
+TEST(ImageFrameBufferPoolStaticTest, BufferCanOutlivePool) {
+  auto pool = ImageFramePool::Create(kWidth, kHeight, kFormat, kKeepCount);
+  auto buffer = pool->GetBuffer();
+  pool = nullptr;
+  buffer = nullptr;
+}
+
+}  // anonymous namespace
+}  // namespace mediapipe

mediapipe/framework/mediapipe_register_type.bzl

@@ -0,0 +1,117 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A rule for registering types with mediapipe.
+
+Example usage:
+    mediapipe_proto_library(
+        name = "foo_proto",
+        srcs = ["foo.proto"],
+    )
+
+    load("//mediapipe/framework:mediapipe_register_type.bzl",
+         "mediapipe_register_type")
+
+    # Creates rules "foo_registration"
+    mediapipe_register_type(
+        base_name = "foo"
+        include_headers = ["mediapipe/framework/formats/foo.proto.h""],
+        types = [
+            "::mediapipe::Foo",
+            "::mediapipe::FooList",
+            "::std::vector<::mediapipe::Foo>",
+        ],
+        deps = [":foo_cc_proto"],
+    )
+
+Args
+  base_name: The base name of the target
+             (name + "_registration" will be created).
+  types: A list of C++ classes to register using MEDIAPIPE_REGISTER_TYPE.
+  deps: A list of cc deps.
+  include_headers: A list of header files that must be included.
+"""
+
+load("//mediapipe/framework/tool:build_defs.bzl", "clean_dep")
+
+def _mediapipe_register_type_generate_cc_impl(ctx):
+    """Generate a cc file that registers types with mediapipe."""
+    file_data_template = '''
+{include_headers}
+#include "mediapipe/framework/type_map.h"
+
+{registration_commands}
+'''
+    header_lines = []
+    for header in ctx.attr.include_headers:
+        header_lines.append('#include "{}"'.format(header))
+    registration_lines = []
+    for registration_type in ctx.attr.types:
+        if " " in registration_type:
+            fail(('registration type "{}" should be fully qualified ' +
+                  "and must not include spaces").format(registration_type))
+        registration_lines.append(
+            "#define TEMP_MP_TYPE {}".format(registration_type),
+        )
+        registration_lines.append(
+            ("MEDIAPIPE_REGISTER_TYPE(\n" +
+             "    TEMP_MP_TYPE,\n" +
+             '    "{}",\n'.format(registration_type) +
+             "    nullptr, nullptr);\n"),
+        )
+        registration_lines.append("#undef TEMP_MP_TYPE")
+
+    file_data = file_data_template.format(
+        include_headers = "\n".join(header_lines),
+        registration_commands = "\n".join(registration_lines),
+    )
+
+    ctx.actions.write(ctx.outputs.output, file_data)
+
+mediapipe_register_type_generate_cc = rule(
+    implementation = _mediapipe_register_type_generate_cc_impl,
+    attrs = {
+        "deps": attr.label_list(),
+        "types": attr.string_list(
+            mandatory = True,
+        ),
+        "include_headers": attr.string_list(
+            mandatory = True,
+        ),
+        "output": attr.output(),
+    },
+)
+
+def mediapipe_register_type(
+        base_name,
+        types,
+        deps = [],
+        include_headers = [],
+        visibility = ["//visibility:public"]):
+    mediapipe_register_type_generate_cc(
+        name = base_name + "_registration_cc",
+        types = types,
+        include_headers = include_headers,
+        output = base_name + "_registration.cc",
+    )
+
+    native.cc_library(
+        name = base_name + "_registration",
+        srcs = [base_name + "_registration.cc"],
+        deps = depset(deps + [
+            clean_dep("//mediapipe/framework:type_map"),
+        ]),
+        visibility = visibility,
+        alwayslink = 1,
+    )

mediapipe/framework/port.h

@@ -52,12 +52,12 @@
 
 // Compile time target platform definitions.
 // Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
-#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0
 #define MEDIAPIPE_OPENGL_ES_20 200
+#define MEDIAPIPE_OPENGL_ES_30 300
 #define MEDIAPIPE_OPENGL_ES_31 310
 
 #if defined(MEDIAPIPE_DISABLE_GPU)
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_OPENGL_ES_VERSION 0
 #define MEDIAPIPE_METAL_ENABLED 0
 #else
 #if defined(MEDIAPIPE_ANDROID)
@@ -71,11 +71,11 @@
 #define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
 #define MEDIAPIPE_METAL_ENABLED 1
 #elif defined(MEDIAPIPE_OSX)
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_OPENGL_ES_VERSION 0
 #define MEDIAPIPE_METAL_ENABLED 1
 #else
-// GPU is not supported on Linux yet.
+// WebGL config.
-#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_30
 #define MEDIAPIPE_METAL_ENABLED 0
 #endif
 #endif

mediapipe/framework/profiler/BUILD

@@ -269,7 +269,7 @@ cc_test(
 
 cc_library(
     name = "profiler_resource_util",
-    srcs = select({
+    srcs = ["profiler_resource_util_common.cc"] + select({
         "//conditions:default": ["profiler_resource_util.cc"],
         "//mediapipe:android": ["profiler_resource_util_android.cc"],
         "//mediapipe:ios": ["profiler_resource_util_ios.cc"],
@@ -288,15 +288,22 @@ cc_library(
     deps = [
         "@com_google_absl//absl/strings",
         "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:statusor",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/flags:flag",
+        "//mediapipe/framework/deps:file_path",
     ] + select({
         "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
         ],
         "//mediapipe:android": [
             "//mediapipe/java/com/google/mediapipe/framework/jni:jni_util",
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:apple": [
+            "//mediapipe/framework/port:file_helpers",
         ],
-        "//mediapipe:apple": [],
     }),
 )
 

mediapipe/framework/profiler/graph_profiler_test.cc

@@ -22,6 +22,7 @@
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/port/statusor.h"
 #include "mediapipe/framework/profiler/test_context_builder.h"
 #include "mediapipe/framework/tool/simulation_clock.h"
 #include "mediapipe/framework/tool/tag_map_helper.h"

mediapipe/framework/profiler/profiler_resource_util.h

@@ -27,6 +27,10 @@ namespace mediapipe {
 // error.
 StatusOr<std::string> GetDefaultTraceLogDirectory();
 
+// Given a log file path, this function provides an absolute path with which
+// it can be accessed as a file.  Enclosing directories are created as needed.
+StatusOr<std::string> PathToLogFile(const std::string& path);
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_FRAMEWORK_PROFILER_PROFILER_RESOURCE_UTIL_H_

mediapipe/framework/profiler/profiler_resource_util_common.cc

@@ -0,0 +1,60 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/flags/flag.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/profiler/profiler_resource_util.h"
+
+// TODO: Move this Android include to port/file_helpers.
+// Also move this from resource_util.cc.
+#ifdef __ANDROID__
+#include "mediapipe/util/android/file/base/filesystem.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+ABSL_FLAG(std::string, log_root_dir, "",
+          "The absolute path to the logging output directory.  If specified, "
+          "log_root_dir will be prepended to each specified log file path.");
+
+#ifdef __ANDROID__
+namespace mediapipe {
+namespace file {
+::mediapipe::Status RecursivelyCreateDir(absl::string_view path) {
+  return RecursivelyCreateDir(path, file::Options());
+}
+}  // namespace file
+}  // namespace mediapipe
+#endif
+
+namespace mediapipe {
+
+::mediapipe::StatusOr<std::string> GetLogDirectory() {
+  if (!FLAGS_log_root_dir.CurrentValue().empty()) {
+    return FLAGS_log_root_dir.CurrentValue();
+  }
+  return GetDefaultTraceLogDirectory();
+}
+
+::mediapipe::StatusOr<std::string> PathToLogFile(const std::string& path) {
+  ASSIGN_OR_RETURN(std::string log_dir, GetLogDirectory());
+  std::string result = file::JoinPath(log_dir, path);
+  MP_RETURN_IF_ERROR(
+      ::mediapipe::file::RecursivelyCreateDir(file::Dirname(result)));
+  return result;
+}
+
+}  // namespace mediapipe

mediapipe/framework/stream_handler/mux_input_stream_handler.cc

@@ -14,6 +14,7 @@
 
 #include "absl/strings/substitute.h"
 #include "absl/synchronization/mutex.h"
+#include "mediapipe/framework/collection_item_id.h"
 #include "mediapipe/framework/input_stream_handler.h"
 #include "mediapipe/framework/port/logging.h"
 
@@ -121,6 +122,16 @@ class MuxInputStreamHandler : public InputStreamHandler {
                             num_packets_dropped, data_stream->Name());
     AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet),
                      stream_is_done);
+
+    // Discard old packets on other streams.
+    // Note that control_stream_id is the last valid id.
+    auto next_timestamp = input_timestamp.NextAllowedInStream();
+    for (CollectionItemId id = input_stream_managers_.BeginId();
+         id < control_stream_id; ++id) {
+      if (id == data_stream_id) continue;
+      auto& other_stream = input_stream_managers_.Get(id);
+      other_stream->ErasePacketsEarlierThan(next_timestamp);
+    }
   }
 
  private:

mediapipe/framework/tool/simulation_clock_executor.cc

@@ -19,11 +19,11 @@
 namespace mediapipe {
 
 SimulationClockExecutor::SimulationClockExecutor(int num_threads)
-    : ThreadPoolExecutor(num_threads), clock_(new SimulationClock()) {}
+    : clock_(new SimulationClock()), executor_(num_threads) {}
 
 void SimulationClockExecutor::Schedule(std::function<void()> task) {
   clock_->ThreadStart();
-  ThreadPoolExecutor::Schedule([this, task] {
+  executor_.Schedule([this, task] {
     clock_->Sleep(absl::ZeroDuration());
     task();
     clock_->ThreadFinish();

mediapipe/framework/tool/simulation_clock_executor.h

@@ -23,7 +23,7 @@ namespace mediapipe {
 // Simulation clock multithreaded executor. This is intended to be used with
 // graphs that are using SimulationClock class to emulate various parts of the
 // graph taking specific time to process the incoming packets.
-class SimulationClockExecutor : public ThreadPoolExecutor {
+class SimulationClockExecutor : public Executor {
  public:
   explicit SimulationClockExecutor(int num_threads);
   void Schedule(std::function<void()> task) override;
@@ -36,6 +36,9 @@ class SimulationClockExecutor : public ThreadPoolExecutor {
  private:
   // SimulationClock instance used by this executor.
   std::shared_ptr<SimulationClock> clock_;
+  // The delegate ThreadPoolExecutor.  This is declared after clock_
+  // so that it is destroyed before clock_.
+  ThreadPoolExecutor executor_;
 };
 
 }  // namespace mediapipe

mediapipe/framework/transitive_protos.bzl

@@ -1,3 +1,17 @@
+# Copyright 2019-2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Extract a cc_library compatible dependency with only the top level proto rules."""
 
 ProtoLibsInfo = provider(fields = ["targets", "out"])

mediapipe/gpu/BUILD

@@ -159,7 +159,7 @@ cc_library(
         "//conditions:default": [],
         "//mediapipe:apple": [
             "-x objective-c++",
-            "-fobjc-arc",
+            "-fobjc-arc",  # enable reference-counting
         ],
     }),
     visibility = ["//visibility:public"],
@@ -407,20 +407,29 @@ cc_library(
     }),
 )
 
+cc_library(
+    name = "gl_texture_buffer_pool",
+    srcs = ["gl_texture_buffer_pool.cc"],
+    hdrs = ["gl_texture_buffer_pool.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":gl_base",
+        ":gl_texture_buffer",
+        ":gpu_buffer",
+        ":gpu_shared_data_header",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_node",
+        "//mediapipe/framework/port:logging",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
 cc_library(
     name = "gpu_buffer_multi_pool",
-    srcs = ["gpu_buffer_multi_pool.cc"] + select({
+    srcs = ["gpu_buffer_multi_pool.cc"],
-        "//conditions:default": [
-            "gl_texture_buffer_pool.cc",
-        ],
-        "//mediapipe:ios": [],
-        "//mediapipe:macos": [
-            "gl_texture_buffer_pool.cc",
-        ],
-    }),
     hdrs = ["gpu_buffer_multi_pool.h"] + select({
         "//conditions:default": [
-            "gl_texture_buffer_pool.h",
         ],
         "//mediapipe:ios": [
             # The inclusions check does not see that this is provided by
@@ -429,7 +438,6 @@ cc_library(
             "pixel_buffer_pool_util.h",
         ],
         "//mediapipe:macos": [
-            "gl_texture_buffer_pool.h",
         ],
     }),
     copts = select({
@@ -452,6 +460,7 @@ cc_library(
     ] + select({
         "//conditions:default": [
             ":gl_texture_buffer",
+            ":gl_texture_buffer_pool",
         ],
         "//mediapipe:ios": [
             ":pixel_buffer_pool_util",
@@ -460,6 +469,7 @@ cc_library(
         "//mediapipe:macos": [
             ":pixel_buffer_pool_util",
             ":gl_texture_buffer",
+            ":gl_texture_buffer_pool",
         ],
     }),
 )

mediapipe/gpu/gl_calculator_helper.cc

@@ -14,14 +14,19 @@
 
 #include "mediapipe/gpu/gl_calculator_helper.h"
 
-#include "absl/memory/memory.h"
+#include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/legacy_calculator_support.h"
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/gpu/gl_calculator_helper_impl.h"
+#include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/gpu_service.h"
 
+#ifdef __APPLE__
+#include "mediapipe/objc/util.h"
+#endif
+
 namespace mediapipe {
 
 GlTexture::GlTexture(GLuint name, int width, int height)

mediapipe/gpu/gl_calculator_helper.h

@@ -15,8 +15,6 @@
 #ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
 #define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
 
-#include <memory>
-
 #include "absl/memory/memory.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_contract.h"
@@ -29,6 +27,7 @@
 #include "mediapipe/gpu/gl_context.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/graph_support.h"
+
 #ifdef __APPLE__
 #include <CoreVideo/CoreVideo.h>
 
@@ -50,6 +49,8 @@ typedef CVOpenGLESTextureRef CVTextureType;
 #endif  // TARGET_OS_OSX
 #endif  // __APPLE__
 
+using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
+
 // TODO: remove this and Process below, or make Process available
 // on Android.
 typedef std::function<void(const GlTexture& src, const GlTexture& dst)>
@@ -121,7 +122,7 @@ class GlCalculatorHelper {
   // where it is supported (iOS, for now) they take advantage of memory sharing
   // between the CPU and GPU, avoiding memory copies.
 
-  // Creates a texture representing an input frame.
+  // Creates a texture representing an input frame, and manages sync token.
   GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
   GlTexture CreateSourceTexture(const ImageFrame& image_frame);
 
@@ -137,7 +138,7 @@ class GlCalculatorHelper {
   void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
                               int* height);
 
-  // Creates a texture representing an output frame.
+  // Creates a texture representing an output frame, and manages sync token.
   // TODO: This should either return errors or a status.
   GlTexture CreateDestinationTexture(
       int output_width, int output_height,
@@ -152,11 +153,22 @@ class GlCalculatorHelper {
 
   GlContext& GetGlContext() const;
 
+  // Check if the calculator helper has been previously initialized.
+  bool Initialized() { return impl_ != nullptr; }
+
  private:
   std::unique_ptr<GlCalculatorHelperImpl> impl_;
 };
 
-// Represents an OpenGL texture.
+// Represents an OpenGL texture, and is a 'view' into the memory pool.
+// It's more like a GlTextureLock, because it's main purpose (in conjunction
+// with the helper): to manage GL sync points in the gl command queue.
+//
+// This class should be the main way to interface with GL memory within a single
+// calculator. This is the preferred way to utilize the memory pool inside of
+// the helper, because GlTexture manages efficiently releasing memory back into
+// the pool. A GPU backed Image can be extracted from the unerlying
+// memory.
 class GlTexture {
  public:
   GlTexture() {}
@@ -170,11 +182,12 @@ class GlTexture {
   GLuint name() const { return name_; }
 
   // Returns a buffer that can be sent to another calculator.
-  // Can be used with GpuBuffer or ImageFrame.
+  // & manages sync token
+  // Can be used with GpuBuffer or ImageFrame or Image
   template <typename T>
   std::unique_ptr<T> GetFrame() const;
 
-  // Releases texture memory
+  // Releases texture memory & manages sync token
   void Release();
 
  private:

mediapipe/gpu/gl_calculator_helper_impl.h

@@ -42,10 +42,10 @@ class GlCalculatorHelperImpl {
       CalculatorContext* calculator_context);
 
   GlTexture CreateSourceTexture(const ImageFrame& image_frame);
-  GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
+  GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer);
 
   // Note: multi-plane support is currently only available on iOS.
-  GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
+  GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane);
 
   // Creates a framebuffer and returns the texture that it is bound to.
   GlTexture CreateDestinationTexture(int output_width, int output_height,

mediapipe/gpu/gl_context_egl.cc

@@ -177,6 +177,27 @@ GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context,
 }
 
 void GlContext::DestroyContext() {
+#ifdef __ANDROID__
+  if (HasContext()) {
+    // Detach the current program to work around b/166322604.
+    auto detach_program = [this] {
+      GlContext::ContextBinding saved_context;
+      GetCurrentContextBinding(&saved_context);
+      // Note: cannot use ThisContextBinding because it calls shared_from_this,
+      // which is not available during destruction.
+      if (eglMakeCurrent(display_, surface_, surface_, context_)) {
+        glUseProgram(0);
+      } else {
+        LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
+                   << std::hex << eglGetError();
+      }
+      return SetCurrentContextBinding(saved_context);
+    };
+    auto status = thread_ ? thread_->Run(detach_program) : detach_program();
+    LOG_IF(ERROR, !status.ok()) << status;
+  }
+#endif  // __ANDROID__
+
   if (thread_) {
     // Delete thread-local storage.
     // TODO: in theory our EglThreadExitCallback should suffice for
@@ -191,18 +212,6 @@ void GlContext::DestroyContext() {
         .IgnoreError();
   }
 
-#ifdef __ANDROID__
-  if (HasContext()) {
-    // Detach the current program to work around b/166322604.
-    if (eglMakeCurrent(display_, surface_, surface_, context_)) {
-      glUseProgram(0);
-    } else {
-      LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
-                 << std::hex << eglGetError();
-    }
-  }
-#endif  // __ANDROID__
-
   // Destroy the context and surface.
   if (IsCurrent()) {
     if (!eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE,

mediapipe/gpu/gpu_buffer.h

@@ -72,7 +72,6 @@ class GpuBuffer {
 
   int width() const;
   int height() const;
-
   GpuBufferFormat format() const;
 
   // Converts to true iff valid.

mediapipe/gpu/gpu_buffer_multi_pool.cc

@@ -44,8 +44,8 @@ static constexpr int kRequestCountScrubInterval = 50;
 
 #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
 
-CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(const BufferSpec& spec,
+CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(
-                                                   CFTimeInterval maxAge) {
+    const GpuBufferMultiPool::BufferSpec& spec, CFTimeInterval maxAge) {
   OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format);
   CHECK_NE(cv_format, -1) << "unsupported pixel format";
   pool_ = MakeCFHolderAdopting(
@@ -90,7 +90,7 @@ std::string CvPixelBufferPoolWrapper::GetDebugString() const {
 void CvPixelBufferPoolWrapper::Flush() { CVPixelBufferPoolFlush(*pool_, 0); }
 
 GpuBufferMultiPool::SimplePool GpuBufferMultiPool::MakeSimplePool(
-    const BufferSpec& spec) {
+    const GpuBufferMultiPool::BufferSpec& spec) {
   return std::make_shared<CvPixelBufferPoolWrapper>(spec,
                                                     kMaxInactiveBufferAge);
 }

mediapipe/gpu/gpu_buffer_multi_pool.h

@@ -40,56 +40,7 @@
 namespace mediapipe {
 
 struct GpuSharedData;
-
+class CvPixelBufferPoolWrapper;
-struct BufferSpec {
-  BufferSpec(int w, int h, GpuBufferFormat f)
-      : width(w), height(h), format(f) {}
-  int width;
-  int height;
-  GpuBufferFormat format;
-};
-
-inline bool operator==(const BufferSpec& lhs, const BufferSpec& rhs) {
-  return lhs.width == rhs.width && lhs.height == rhs.height &&
-         lhs.format == rhs.format;
-}
-inline bool operator!=(const BufferSpec& lhs, const BufferSpec& rhs) {
-  return !operator==(lhs, rhs);
-}
-
-// This generates a "rol" instruction with both Clang and GCC.
-static inline std::size_t RotateLeft(std::size_t x, int n) {
-  return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
-}
-
-struct BufferSpecHash {
-  std::size_t operator()(const mediapipe::BufferSpec& spec) const {
-    // Width and height are expected to be smaller than half the width of
-    // size_t. We can combine them into a single integer, and then use
-    // std::hash, which is what go/hashing recommends for hashing numbers.
-    constexpr int kWidth = std::numeric_limits<size_t>::digits;
-    return std::hash<std::size_t>{}(
-        spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
-        RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
-  }
-};
-
-#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
-class CvPixelBufferPoolWrapper {
- public:
-  CvPixelBufferPoolWrapper(const BufferSpec& spec, CFTimeInterval maxAge);
-  GpuBuffer GetBuffer(std::function<void(void)> flush);
-
-  int GetBufferCount() const { return count_; }
-  std::string GetDebugString() const;
-
-  void Flush();
-
- private:
-  CFHolder<CVPixelBufferPoolRef> pool_;
-  int count_ = 0;
-};
-#endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
 
 class GpuBufferMultiPool {
  public:
@@ -114,6 +65,31 @@ class GpuBufferMultiPool {
   void FlushTextureCaches();
 #endif  // defined(__APPLE__)
 
+  // This generates a "rol" instruction with both Clang and GCC.
+  inline static std::size_t RotateLeft(std::size_t x, int n) {
+    return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
+  }
+
+  struct BufferSpec {
+    BufferSpec(int w, int h, mediapipe::GpuBufferFormat f)
+        : width(w), height(h), format(f) {}
+    int width;
+    int height;
+    mediapipe::GpuBufferFormat format;
+  };
+
+  struct BufferSpecHash {
+    std::size_t operator()(const BufferSpec& spec) const {
+      // Width and height are expected to be smaller than half the width of
+      // size_t. We can combine them into a single integer, and then use
+      // std::hash, which is what go/hashing recommends for hashing numbers.
+      constexpr int kWidth = std::numeric_limits<size_t>::digits;
+      return std::hash<std::size_t>{}(
+          spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
+          RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
+    }
+  };
+
  private:
 #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   using SimplePool = std::shared_ptr<CvPixelBufferPoolWrapper>;
@@ -175,6 +151,35 @@ class GpuBufferMultiPool {
 #endif  // defined(__APPLE__)
 };
 
+#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
+class CvPixelBufferPoolWrapper {
+ public:
+  CvPixelBufferPoolWrapper(const GpuBufferMultiPool::BufferSpec& spec,
+                           CFTimeInterval maxAge);
+  GpuBuffer GetBuffer(std::function<void(void)> flush);
+
+  int GetBufferCount() const { return count_; }
+  std::string GetDebugString() const;
+
+  void Flush();
+
+ private:
+  CFHolder<CVPixelBufferPoolRef> pool_;
+  int count_ = 0;
+};
+#endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
+
+// BufferSpec equality operators
+inline bool operator==(const GpuBufferMultiPool::BufferSpec& lhs,
+                       const GpuBufferMultiPool::BufferSpec& rhs) {
+  return lhs.width == rhs.width && lhs.height == rhs.height &&
+         lhs.format == rhs.format;
+}
+inline bool operator!=(const GpuBufferMultiPool::BufferSpec& lhs,
+                       const GpuBufferMultiPool::BufferSpec& rhs) {
+  return !operator==(lhs, rhs);
+}
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_

mediapipe/gpu/gpu_shared_data_internal.cc

@@ -110,6 +110,7 @@ GpuResources::~GpuResources() {
   std::string node_type = node->GetCalculatorState().CalculatorType();
   std::string context_key;
 
+#ifndef __EMSCRIPTEN__
   // TODO Allow calculators to request a separate context.
   // For now, white-list a few calculators to run in their own context.
   bool gets_own_context = (node_type == "ImageFrameToGpuBufferCalculator") ||
@@ -126,6 +127,10 @@ GpuResources::~GpuResources() {
   } else {
     context_key = absl::StrCat("auto:", node_id);
   }
+#else
+  // On Emscripten we currently do not support multiple contexts.
+  context_key = SharedContextKey();
+#endif  // !__EMSCRIPTEN__
   node_key_[node_id] = context_key;
 
   ASSIGN_OR_RETURN(std::shared_ptr<GlContext> context,

mediapipe/graphs/face_effect/BUILD

@@ -0,0 +1,43 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_binary_graph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "face_effect_gpu_deps",
+    deps = [
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:immediate_mux_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/graphs/face_effect/subgraphs:single_face_smooth_landmark_gpu",
+        "//mediapipe/modules/face_geometry",
+        "//mediapipe/modules/face_geometry:effect_renderer_calculator",
+        "//mediapipe/modules/face_geometry:env_generator_calculator",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "face_effect_gpu_binary_graph",
+    graph = "face_effect_gpu.pbtxt",
+    output_name = "face_effect_gpu.binarypb",
+    deps = [":face_effect_gpu_deps"],
+)

mediapipe/graphs/face_effect/data/BUILD

@@ -0,0 +1,36 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+encode_binary_proto(
+    name = "glasses",
+    input = "glasses.pbtxt",
+    message_type = "mediapipe.face_geometry.Mesh3d",
+    output = "glasses.binarypb",
+    deps = [
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_proto",
+    ],
+)
+
+# `.pngblob` is used instead of `.png` to prevent iOS build from preprocessing the image.
+# OpenCV is unable to read a PNG file preprocessed by the iOS build.
+exports_files([
+    "facepaint.pngblob",
+    "glasses.pngblob",
+])

mediapipe/graphs/face_effect/face_effect_gpu.pbtxt

@@ -0,0 +1,145 @@
+# MediaPipe graph that applies a face effect to the input video stream.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Boolean flag, which indicates whether the Facepaint effect is selected. (bool)
+#
+# If `true`, the Facepaint effect will be rendered.
+# If `false`, the Glasses effect will be rendered.
+input_stream: "is_facepaint_effect_selected"
+
+# Output image with rendered results. (GpuBuffer)
+output_stream: "output_video"
+
+# A list of geometry data for a single detected face.
+#
+# NOTE: there will not be an output packet in this stream for this particular
+# timestamp if none of faces detected.
+#
+# (std::vector<face_geometry::FaceGeometry>)
+output_stream: "multi_face_geometry"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Generates an environment that describes the current virtual scene.
+node {
+  calculator: "FaceGeometryEnvGeneratorCalculator"
+  output_side_packet: "ENVIRONMENT:environment"
+  node_options: {
+    [type.googleapis.com/mediapipe.FaceGeometryEnvGeneratorCalculatorOptions] {
+      environment: {
+        origin_point_location: TOP_LEFT_CORNER
+        perspective_camera: {
+          vertical_fov_degrees: 63.0  # 63 degrees
+          near: 1.0  # 1cm
+          far: 10000.0  # 100m
+        }
+      }
+    }
+  }
+}
+
+# Subgraph that detects a single face and corresponding landmarks. The landmarks
+# are also "smoothed" to achieve better visual results.
+node {
+  calculator: "SingleFaceSmoothLandmarkGpu"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+}
+
+# Extracts the throttled input video frame dimensions as a separate packet.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:throttled_input_video"
+  output_stream: "SIZE:input_video_size"
+}
+
+# Subgraph that computes face geometry from landmarks for a single face.
+node {
+  calculator: "FaceGeometry"
+  input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+  input_stream: "IMAGE_SIZE:input_video_size"
+  input_side_packet: "ENVIRONMENT:environment"
+  output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+}
+
+# Decides whether to render the Facepaint effect based on the
+# `is_facepaint_effect_selected` flag value.
+node {
+  calculator: "GateCalculator"
+  input_stream: "throttled_input_video"
+  input_stream: "multi_face_geometry"
+  input_stream: "ALLOW:is_facepaint_effect_selected"
+  output_stream: "facepaint_effect_throttled_input_video"
+  output_stream: "facepaint_effect_multi_face_geometry"
+}
+
+# Renders the Facepaint effect.
+node {
+  calculator: "FaceGeometryEffectRendererCalculator"
+  input_side_packet: "ENVIRONMENT:environment"
+  input_stream: "IMAGE_GPU:facepaint_effect_throttled_input_video"
+  input_stream: "MULTI_FACE_GEOMETRY:facepaint_effect_multi_face_geometry"
+  output_stream: "IMAGE_GPU:facepaint_effect_output_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
+      effect_texture_path: "mediapipe/graphs/face_effect/data/facepaint.pngblob"
+    }
+  }
+}
+
+# Decides whether to render the Glasses effect based on the
+# `is_facepaint_effect_selected` flag value.
+node {
+  calculator: "GateCalculator"
+  input_stream: "throttled_input_video"
+  input_stream: "multi_face_geometry"
+  input_stream: "DISALLOW:is_facepaint_effect_selected"
+  output_stream: "glasses_effect_throttled_input_video"
+  output_stream: "glasses_effect_multi_face_geometry"
+}
+
+# Renders the Glasses effect.
+node {
+  calculator: "FaceGeometryEffectRendererCalculator"
+  input_side_packet: "ENVIRONMENT:environment"
+  input_stream: "IMAGE_GPU:glasses_effect_throttled_input_video"
+  input_stream: "MULTI_FACE_GEOMETRY:glasses_effect_multi_face_geometry"
+  output_stream: "IMAGE_GPU:glasses_effect_output_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.FaceGeometryEffectRendererCalculatorOptions] {
+      effect_texture_path: "mediapipe/graphs/face_effect/data/glasses.pngblob"
+      effect_mesh_3d_path: "mediapipe/graphs/face_effect/data/glasses.binarypb"
+    }
+  }
+}
+
+# Decides which of the Facepaint or the Glasses rendered results should be sent
+# as the output GPU frame.
+node {
+  calculator: "ImmediateMuxCalculator"
+  input_stream: "facepaint_effect_output_video"
+  input_stream: "glasses_effect_output_video"
+  output_stream: "output_video"
+}
+

mediapipe/graphs/face_effect/subgraphs/BUILD

@@ -0,0 +1,36 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_simple_subgraph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_simple_subgraph(
+    name = "single_face_smooth_landmark_gpu",
+    graph = "single_face_smooth_landmark_gpu.pbtxt",
+    register_as = "SingleFaceSmoothLandmarkGpu",
+    deps = [
+        "//mediapipe/calculators/core:concatenate_vector_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:split_vector_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
+        "//mediapipe/modules/face_landmark:face_landmark_front_gpu",
+    ],
+)

mediapipe/graphs/face_effect/subgraphs/single_face_smooth_landmark_gpu.pbtxt

@@ -0,0 +1,84 @@
+# MediaPipe subgraph that detects a single face and corresponding landmarks on
+# a input GPU image. The landmarks are also "smoothed" to achieve better visual
+# results.
+
+type: "SingleFaceSmoothLandmarkGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:input_image"
+
+# Collection of detected/predicted faces, each represented as a list of face
+# landmarks. However, the size of this collection is always 1 because of the
+# single-face use in this graph. The decision to wrap the landmark list into a
+# collection was made to simplify passing the result into the `FaceGeometry`
+# subgraph. (std::vector<NormalizedLandmarkList>)
+#
+# NOTE: there will not be an output packet in the LANDMARKS stream for this
+# particular timestamp if none of faces detected. However, the MediaPipe
+# framework will internally inform the downstream calculators of the absence of
+# this packet so that they don't wait for it unnecessarily.
+output_stream: "LANDMARKS:multi_face_smooth_landmarks"
+
+# Creates a packet to inform the `FaceLandmarkFrontGpu` subgraph to detect at
+# most 1 face.
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:num_faces"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { int_value: 1 }
+    }
+  }
+}
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontGpu"
+  input_stream: "IMAGE:input_image"
+  input_side_packet: "NUM_FACES:num_faces"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+}
+
+# Extracts the detected face landmark list from a collection.
+node {
+  calculator: "SplitNormalizedLandmarkListVectorCalculator"
+  input_stream: "multi_face_landmarks"
+  output_stream: "face_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.SplitVectorCalculatorOptions] {
+      ranges: { begin: 0 end: 1 }
+      element_only: true
+    }
+  }
+}
+
+# Extracts the input image frame dimensions as a separate packet.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "SIZE:input_image_size"
+}
+
+# Applies smoothing to the single face landmarks.
+node {
+  calculator: "LandmarksSmoothingCalculator"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  input_stream: "IMAGE_SIZE:input_image_size"
+  output_stream: "NORM_FILTERED_LANDMARKS:face_smooth_landmarks"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] {
+      velocity_filter: {
+        window_size: 5
+        velocity_scale: 20.0
+      }
+    }
+  }
+}
+
+# Puts the single face smooth landmarks back into a collection to simplify
+# passing the result into the `FaceGeometry` subgraph.
+node {
+  calculator: "ConcatenateLandmarListVectorCalculator"
+  input_stream: "face_smooth_landmarks"
+  output_stream: "multi_face_smooth_landmarks"
+}

mediapipe/graphs/object_detection_3d/calculators/BUILD

@@ -16,7 +16,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library"
 
 licenses(["notice"])
 
-package(default_visibility = ["//visibility:private"])
+package(default_visibility = ["//visibility:public"])
 
 proto_library(
     name = "object_proto",

mediapipe/graphs/pose_tracking/BUILD

@@ -26,7 +26,7 @@ cc_library(
     deps = [
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
         "//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_gpu",
         "//mediapipe/modules/pose_landmark:pose_landmark_upper_body_gpu",
     ],
@@ -44,7 +44,7 @@ cc_library(
     deps = [
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/image:image_properties_calculator",
-        "//mediapipe/graphs/pose_tracking/calculators:landmarks_smoothing_calculator",
+        "//mediapipe/calculators/util:landmarks_smoothing_calculator",
         "//mediapipe/graphs/pose_tracking/subgraphs:upper_body_pose_renderer_cpu",
         "//mediapipe/modules/pose_landmark:pose_landmark_upper_body_cpu",
     ],

mediapipe/java/com/google/mediapipe/components/FrameProcessor.java

@@ -233,19 +233,20 @@ public class FrameProcessor implements TextureFrameProcessor, AudioDataProcessor
    *
    * @param inputStream the graph input stream that will receive input audio samples.
    * @param outputStream the output stream from which output audio samples will be produced.
-   * @param numChannels the number of audio channels in the input audio stream.
+   * @param numInputChannels the number of audio channels in the input audio stream.
+   * @param numOutputChannels the number of audio channels in the output audio stream. If there is
+   *                          no output stream, set this to zero.
    * @param audioSampleRateInHz the sample rate for audio samples in hertz (Hz).
    */
   public void addAudioStreams(
       @Nullable String inputStream,
       @Nullable String outputStream,
-      int numChannels,
+      int numInputChannels,
+      int numOutputChannels,
       double audioSampleRateInHz) {
     audioInputStream = inputStream;
     audioOutputStream = outputStream;
-    numAudioChannels = numChannels;
+    numAudioChannels = numInputChannels;
-    int audioChannelMask =
-        numAudioChannels == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO;
     audioSampleRate = audioSampleRateInHz;
 
     if (audioInputStream != null) {
@@ -254,11 +255,13 @@ public class FrameProcessor implements TextureFrameProcessor, AudioDataProcessor
     }
 
     if (audioOutputStream != null) {
+      int outputAudioChannelMask =
+          numOutputChannels == 2 ? AudioFormat.CHANNEL_IN_STEREO : AudioFormat.CHANNEL_IN_MONO;
       AudioFormat audioFormat =
           new AudioFormat.Builder()
               .setEncoding(AUDIO_ENCODING)
               .setSampleRate((int) audioSampleRate)
-              .setChannelMask(audioChannelMask)
+              .setChannelMask(outputAudioChannelMask)
               .build();
       mediapipeGraph.addPacketCallback(
           audioOutputStream,

mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl

@@ -1,3 +1,17 @@
+# Copyright 2019-2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Generate MediaPipe AAR including different variants of .so in jni folder.
 
 Usage:

mediapipe/models/README.md

@@ -1,69 +1 @@
-## MediaPipe Models
+Please see https://solutions.mediapipe.dev/models for more description and model cards.
-
-### Face Detection
-  * For front-facing/selfie cameras: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite)
-  * For back-facing cameras: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_back.tflite)
-  * [Model page](https://sites.google.com/corp/view/perception-cv4arvr/blazeface)
-  * Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://arxiv.org/abs/1907.05047)
-  * [Model card](https://mediapipe.page.link/blazeface-mc)
-
-### Face Mesh
-  * Face detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite) (see above)
-  * 3D face landmarks: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_landmark.tflite), [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
-  * [Model page](https://sites.google.com/corp/view/perception-cv4arvr/facemesh)
-  * Paper: ["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724)
-  * [Google AI Blog post](https://ai.googleblog.com/2019/03/real-time-ar-self-expression-with.html)
-  * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
-  * [Model card](https://mediapipe.page.link/facemesh-mc)
-
-### Hand Detection and Tracking
-  * Palm detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite), [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
-  * 3D hand landmarks: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite), [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
-  * [Google AI Blog post](https://ai.googleblog.com/2019/08/on-device-real-time-hand-tracking-with.html)
-  * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
-  * [Model card](https://mediapipe.page.link/handmc)
-
-### Iris
-  * Iris landmarks:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/iris_landmark.tflite)
-  * Paper:
-    [Real-time Pupil Tracking from Monocular Video for Digital Puppetry](https://arxiv.org/abs/2006.11341)
-    ([presentation](https://youtu.be/cIhXkiiapQI))
-  * Google AI Blog:
-    [MediaPipe Iris: Real-time Eye Tracking and Depth Estimation](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html)
-  * [Model card](https://mediapipe.page.link/iris-mc)
-
-### Pose
-  * Pose detection:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
-  * Upper-body pose landmarks:
-    [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_upper_body.tflite)
-  * Paper:
-    [BlazePose: On-device Real-time Body Pose Tracking](https://arxiv.org/abs/2006.10204)
-    ([presentation](https://youtu.be/YPpUOTRn5tA))
-  * Google AI Blog:
-    [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
-  * [Model card](https://mediapipe.page.link/blazepose-mc)
-
-### Hair Segmentation
-  * [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
-  * [Model page](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation)
-  * Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://arxiv.org/abs/1907.06740)
-  * [Model card](https://mediapipe.page.link/hairsegmentation-mc)
-
-### Objectron (3D Object Detection)
-  * Shoes: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_sneakers.tflite)
-  * Chairs: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_3d_chair.tflite)
-  * [Google AI Blog post](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html)
-
-### Object Detection
-* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
-* See [here](object_detection_saved_model/README.md) for model details.
-
-### KNIFT (Keypoint Neural Invariant Feature Transform)
-  * Up to 200 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
-  * Up to 400 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
-  * Up to 1000 keypoints: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
-  * [Google Developers Blog post](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
-  * [Model card](https://mediapipe.page.link/knift-mc)
-

mediapipe/modules/face_geometry/BUILD

@@ -0,0 +1,116 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+mediapipe_simple_subgraph(
+    name = "face_geometry",
+    graph = "face_geometry.pbtxt",
+    register_as = "FaceGeometry",
+    deps = [
+        ":geometry_pipeline_calculator",
+    ],
+)
+
+mediapipe_proto_library(
+    name = "effect_renderer_calculator_proto",
+    srcs = ["effect_renderer_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+cc_library(
+    name = "effect_renderer_calculator",
+    srcs = ["effect_renderer_calculator.cc"],
+    deps = [
+        ":effect_renderer_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/gpu:gl_calculator_helper",
+        "//mediapipe/gpu:gpu_buffer",
+        "//mediapipe/modules/face_geometry/libs:effect_renderer",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
+        "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/types:optional",
+    ],
+    alwayslink = 1,
+)
+
+mediapipe_proto_library(
+    name = "env_generator_calculator_proto",
+    srcs = ["env_generator_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/modules/face_geometry/protos:environment_proto",
+    ],
+)
+
+cc_library(
+    name = "env_generator_calculator",
+    srcs = ["env_generator_calculator.cc"],
+    deps = [
+        ":env_generator_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+mediapipe_proto_library(
+    name = "geometry_pipeline_calculator_proto",
+    srcs = ["geometry_pipeline_calculator.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+    ],
+)
+
+cc_library(
+    name = "geometry_pipeline_calculator",
+    srcs = ["geometry_pipeline_calculator.cc"],
+    deps = [
+        ":geometry_pipeline_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:statusor",
+        "//mediapipe/modules/face_geometry/libs:geometry_pipeline",
+        "//mediapipe/modules/face_geometry/libs:validation_utils",
+        "//mediapipe/modules/face_geometry/protos:environment_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
+        "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)

mediapipe/modules/face_geometry/README.md

@@ -0,0 +1,18 @@
+# face_geometry
+
+Protos|Details
+:--- | :---
+[`face_geometry.Environment`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/environment.proto)| Describes an environment; includes the camera frame origin point location as well as virtual camera parameters.
+[`face_geometry.GeometryPipelineMetadata`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto)| Describes metadata needed to estimate face geometry based on the face landmark module result.
+[`face_geometry.FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto)| Describes geometry data for a single face; includes a face mesh surface and a face pose in a given environment.
+[`face_geometry.Mesh3d`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/mesh_3d.proto)| Describes a 3D mesh surface.
+
+Calculators|Details
+:--- | :---
+[`FaceGeometryEnvGeneratorCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/env_generator_calculator.cc)| Generates an environment that describes a virtual scene.
+[`FaceGeometryPipelineCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc)| Extracts face geometry for multiple faces from a vector of landmark lists.
+[`FaceGeometryEffectRendererCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc)| Renders a face effect.
+
+Subgraphs|Details
+:--- | :---
+[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts face geometry from landmarks for multiple faces.

mediapipe/modules/face_geometry/data/BUILD

@@ -0,0 +1,37 @@
+# Copyright 2020 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+encode_binary_proto(
+    name = "geometry_pipeline_metadata",
+    input = "geometry_pipeline_metadata.pbtxt",
+    message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
+    output = "geometry_pipeline_metadata.binarypb",
+    deps = [
+        "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
+    ],
+)
+
+# These canonical face model files are not meant to be used in runtime, but rather for asset
+# creation and/or reference.
+exports_files([
+    "canonical_face_model.fbx",
+    "canonical_face_model.obj",
+    "canonical_face_model_uv_visualization.png",
+])

mediapipe/modules/face_geometry/effect_renderer_calculator.cc

@@ -0,0 +1,284 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"       // NOTYPO
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"  // NOTYPO
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"    // NOTYPO
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/modules/face_geometry/effect_renderer_calculator.pb.h"
+#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
+#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
+#include "mediapipe/util/resource_util.h"
+
+namespace mediapipe {
+namespace {
+
+static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
+static constexpr char kImageGpuTag[] = "IMAGE_GPU";
+static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
+
+// A calculator that renders a visual effect for multiple faces.
+//
+// Inputs:
+//   IMAGE_GPU (`GpuBuffer`, required):
+//     A buffer containing input image.
+//
+//   MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, optional):
+//     A vector of face geometry data.
+//
+//     If absent, the input GPU buffer is copied over into the output GPU buffer
+//     without any effect being rendered.
+//
+// Input side packets:
+//   ENVIRONMENT (`face_geometry::Environment`, required)
+//     Describes an environment; includes the camera frame origin point location
+//     as well as virtual camera parameters.
+//
+// Output:
+//   IMAGE_GPU (`GpuBuffer`, required):
+//     A buffer with a visual effect being rendered for multiple faces.
+//
+// Options:
+//   effect_texture_path (`string`, required):
+//     Defines a path for the visual effect texture file. The effect texture is
+//     later rendered on top of the effect mesh.
+//
+//     The texture file format must be supported by the OpenCV image decoder. It
+//     must also define either an RGB or an RGBA texture.
+//
+//   effect_mesh_3d_path (`string`, optional):
+//     Defines a path for the visual effect mesh 3D file. The effect mesh is
+//     later "attached" to the face and is driven by the face pose
+//     transformation matrix.
+//
+//     The mesh 3D file format must be the binary `face_geometry.Mesh3d` proto.
+//
+//     If is not present, the runtime face mesh will be used as the effect mesh
+//     - this mode is handy for facepaint effects.
+//
+class EffectRendererCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc))
+        << "Failed to update contract for the GPU helper!";
+
+    cc->InputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set<face_geometry::Environment>();
+    cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+    cc->Inputs()
+        .Tag(kMultiFaceGeometryTag)
+        .Set<std::vector<face_geometry::FaceGeometry>>();
+    cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
+
+    return mediapipe::GlCalculatorHelper::UpdateContract(cc);
+  }
+
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
+
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc))
+        << "Failed to open the GPU helper!";
+    return gpu_helper_.RunInGlContext([&]() -> mediapipe::Status {
+      const auto& options =
+          cc->Options<FaceGeometryEffectRendererCalculatorOptions>();
+
+      const auto& environment = cc->InputSidePackets()
+                                    .Tag(kEnvironmentTag)
+                                    .Get<face_geometry::Environment>();
+
+      MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
+          << "Invalid environment!";
+
+      absl::optional<face_geometry::Mesh3d> effect_mesh_3d;
+      if (options.has_effect_mesh_3d_path()) {
+        ASSIGN_OR_RETURN(effect_mesh_3d,
+                         ReadMesh3dFromFile(options.effect_mesh_3d_path()),
+                         _ << "Failed to read the effect 3D mesh from file!");
+
+        MP_RETURN_IF_ERROR(face_geometry::ValidateMesh3d(*effect_mesh_3d))
+            << "Invalid effect 3D mesh!";
+      }
+
+      ASSIGN_OR_RETURN(ImageFrame effect_texture,
+                       ReadTextureFromFile(options.effect_texture_path()),
+                       _ << "Failed to read the effect texture from file!");
+
+      ASSIGN_OR_RETURN(effect_renderer_,
+                       CreateEffectRenderer(environment, effect_mesh_3d,
+                                            std::move(effect_texture)),
+                       _ << "Failed to create the effect renderer!");
+
+      return mediapipe::OkStatus();
+    });
+  }
+
+  mediapipe::Status Process(CalculatorContext* cc) override {
+    // The `IMAGE_GPU` stream is required to have a non-empty packet. In case
+    // this requirement is not met, there's nothing to be processed at the
+    // current timestamp.
+    if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
+      return mediapipe::OkStatus();
+    }
+
+    return gpu_helper_.RunInGlContext([this, cc]() -> mediapipe::Status {
+      const auto& input_gpu_buffer =
+          cc->Inputs().Tag(kImageGpuTag).Get<GpuBuffer>();
+
+      GlTexture input_gl_texture =
+          gpu_helper_.CreateSourceTexture(input_gpu_buffer);
+
+      GlTexture output_gl_texture = gpu_helper_.CreateDestinationTexture(
+          input_gl_texture.width(), input_gl_texture.height());
+
+      std::vector<face_geometry::FaceGeometry> empty_multi_face_geometry;
+      const auto& multi_face_geometry =
+          cc->Inputs().Tag(kMultiFaceGeometryTag).IsEmpty()
+              ? empty_multi_face_geometry
+              : cc->Inputs()
+                    .Tag(kMultiFaceGeometryTag)
+                    .Get<std::vector<face_geometry::FaceGeometry>>();
+
+      // Validate input multi face geometry data.
+      for (const face_geometry::FaceGeometry& face_geometry :
+           multi_face_geometry) {
+        MP_RETURN_IF_ERROR(face_geometry::ValidateFaceGeometry(face_geometry))
+            << "Invalid face geometry!";
+      }
+
+      MP_RETURN_IF_ERROR(effect_renderer_->RenderEffect(
+          multi_face_geometry, input_gl_texture.width(),
+          input_gl_texture.height(), input_gl_texture.target(),
+          input_gl_texture.name(), output_gl_texture.target(),
+          output_gl_texture.name()))
+          << "Failed to render the effect!";
+
+      std::unique_ptr<GpuBuffer> output_gpu_buffer =
+          output_gl_texture.GetFrame<GpuBuffer>();
+
+      cc->Outputs()
+          .Tag(kImageGpuTag)
+          .AddPacket(mediapipe::Adopt<GpuBuffer>(output_gpu_buffer.release())
+                         .At(cc->InputTimestamp()));
+
+      output_gl_texture.Release();
+      input_gl_texture.Release();
+
+      return mediapipe::OkStatus();
+    });
+  }
+
+  ~EffectRendererCalculator() {
+    gpu_helper_.RunInGlContext([this]() { effect_renderer_.reset(); });
+  }
+
+ private:
+  static mediapipe::StatusOr<ImageFrame> ReadTextureFromFile(
+      const std::string& texture_path) {
+    ASSIGN_OR_RETURN(std::string texture_blob,
+                     ReadContentBlobFromFile(texture_path),
+                     _ << "Failed to read texture blob from file!");
+
+    // Use OpenCV image decoding functionality to finish reading the texture.
+    std::vector<char> texture_blob_vector(texture_blob.begin(),
+                                          texture_blob.end());
+    cv::Mat decoded_mat =
+        cv::imdecode(texture_blob_vector, cv::IMREAD_UNCHANGED);
+
+    RET_CHECK(decoded_mat.type() == CV_8UC3 || decoded_mat.type() == CV_8UC4)
+        << "Texture must have `char` as the underlying type and "
+           "must have either 3 or 4 channels!";
+
+    ImageFormat::Format image_format = ImageFormat::UNKNOWN;
+    cv::Mat output_mat;
+    switch (decoded_mat.channels()) {
+      case 3:
+        image_format = ImageFormat::SRGB;
+        cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
+        break;
+
+      case 4:
+        image_format = ImageFormat::SRGBA;
+        cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGRA2RGBA);
+        break;
+
+      default:
+        RET_CHECK_FAIL()
+            << "Unexpected number of channels; expected 3 or 4, got "
+            << decoded_mat.channels() << "!";
+    }
+
+    ImageFrame output_image_frame(image_format, output_mat.size().width,
+                                  output_mat.size().height,
+                                  ImageFrame::kGlDefaultAlignmentBoundary);
+
+    output_mat.copyTo(formats::MatView(&output_image_frame));
+
+    return output_image_frame;
+  }
+
+  static mediapipe::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
+      const std::string& mesh_3d_path) {
+    ASSIGN_OR_RETURN(std::string mesh_3d_blob,
+                     ReadContentBlobFromFile(mesh_3d_path),
+                     _ << "Failed to read mesh 3D blob from file!");
+
+    face_geometry::Mesh3d mesh_3d;
+    RET_CHECK(mesh_3d.ParseFromString(mesh_3d_blob))
+        << "Failed to parse a mesh 3D proto from a binary blob!";
+
+    return mesh_3d;
+  }
+
+  static mediapipe::StatusOr<std::string> ReadContentBlobFromFile(
+      const std::string& unresolved_path) {
+    ASSIGN_OR_RETURN(std::string resolved_path,
+                     mediapipe::PathToResourceAsFile(unresolved_path),
+                     _ << "Failed to resolve path! Path = " << unresolved_path);
+
+    std::string content_blob;
+    MP_RETURN_IF_ERROR(
+        mediapipe::GetResourceContents(resolved_path, &content_blob))
+        << "Failed to read content blob! Resolved path = " << resolved_path;
+
+    return content_blob;
+  }
+
+  mediapipe::GlCalculatorHelper gpu_helper_;
+  std::unique_ptr<face_geometry::EffectRenderer> effect_renderer_;
+};
+
+}  // namespace
+
+using FaceGeometryEffectRendererCalculator = EffectRendererCalculator;
+
+REGISTER_CALCULATOR(FaceGeometryEffectRendererCalculator);
+
+}  // namespace mediapipe

mediapipe/modules/face_geometry/effect_renderer_calculator.proto

@@ -0,0 +1,46 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+
+message FaceGeometryEffectRendererCalculatorOptions {
+  extend CalculatorOptions {
+    optional FaceGeometryEffectRendererCalculatorOptions ext = 323693808;
+  }
+
+  // Defines a path for the visual effect texture file. The effect texture is
+  // later rendered on top of the effect mesh.
+  //
+  // Please be aware about the difference between the CPU texture memory layout
+  // and the GPU texture sampler coordinate space. This renderer follows
+  // conventions discussed here: https://open.gl/textures
+  //
+  // The texture file format must be supported by the OpenCV image decoder. It
+  // must also define either an RGB or an RGBA texture.
+  optional string effect_texture_path = 1;
+
+  // Defines a path for the visual effect mesh 3D file. The effect mesh is later
+  // "attached" to the face and is driven by the face pose transformation
+  // matrix.
+  //
+  // The mesh 3D file format must be the binary `face_system.Mesh3d` proto.
+  //
+  // If is not present, the runtime face mesh will be used as the effect mesh
+  // - this mode is handy for facepaint effects.
+  optional string effect_mesh_3d_path = 2;
+}

mediapipe/modules/face_geometry/env_generator_calculator.cc

@@ -0,0 +1,81 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/modules/face_geometry/env_generator_calculator.pb.h"
+#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
+#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
+
+namespace mediapipe {
+namespace {
+
+static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
+
+// A calculator that generates an environment, which describes a virtual scene.
+//
+// Output side packets:
+//   ENVIRONMENT (`face_geometry::Environment`, required)
+//     Describes an environment; includes the camera frame origin point location
+//     as well as virtual camera parameters.
+//
+// Options:
+//   environment (`face_geometry.Environment`, required):
+//     Defines an environment to be packed as the output side packet.
+//
+//     Must be valid (for details, please refer to the proto message definition
+//     comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
+//
+class EnvGeneratorCalculator : public CalculatorBase {
+ public:
+  static mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->OutputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set<face_geometry::Environment>();
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(mediapipe::TimestampDiff(0));
+
+    const face_geometry::Environment& environment =
+        cc->Options<FaceGeometryEnvGeneratorCalculatorOptions>().environment();
+
+    MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
+        << "Invalid environment!";
+
+    cc->OutputSidePackets()
+        .Tag(kEnvironmentTag)
+        .Set(mediapipe::MakePacket<face_geometry::Environment>(environment));
+
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Process(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
+  }
+
+  mediapipe::Status Close(CalculatorContext* cc) override {
+    return mediapipe::OkStatus();
+  }
+};
+
+}  // namespace
+
+using FaceGeometryEnvGeneratorCalculator = EnvGeneratorCalculator;
+
+REGISTER_CALCULATOR(FaceGeometryEnvGeneratorCalculator);
+
+}  // namespace mediapipe

mediapipe/modules/face_geometry/env_generator_calculator.proto

@@ -0,0 +1,32 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator_options.proto";
+import "mediapipe/modules/face_geometry/protos/environment.proto";
+
+message FaceGeometryEnvGeneratorCalculatorOptions {
+  extend CalculatorOptions {
+    optional FaceGeometryEnvGeneratorCalculatorOptions ext = 323693810;
+  }
+
+  // Defines an environment to be packed as the output side packet.
+  //
+  // Must be valid (for details, please refer to the proto message definition
+  // comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
+  optional face_geometry.Environment environment = 1;
+}

mediapipe/modules/face_geometry/face_geometry.pbtxt

@@ -0,0 +1,52 @@
+# MediaPipe graph to extract face geometry from landmarks for multiple faces.
+#
+# It is required that "geometry_pipeline_metadata.binarypb" is available at
+# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
+# path during execution.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "FaceGeometry"
+#     input_stream: "IMAGE_SIZE:image_size"
+#     input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+#     input_side_packet: "ENVIRONMENT:environment"
+#     output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+#   }
+
+type: "FaceGeometry"
+
+# The size of the input frame. The first element of the pair is the frame width;
+# the other one is the frame height.
+#
+# The face landmarks should have been detected on a frame with the same
+# ratio. If used as-is, the resulting face geometry visualization should be
+# happening on a frame with the same ratio as well.
+#
+# (std::pair<int, int>)
+input_stream: "IMAGE_SIZE:image_size"
+
+# Collection of detected/predicted faces, each represented as a list of face
+# landmarks. (std::vector<NormalizedLandmarkList>)
+input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+
+# Environment that describes the current virtual scene.
+# (face_geometry::Environment)
+input_side_packet: "ENVIRONMENT:environment"
+
+# A list of geometry data for each detected face.
+# (std::vector<face_geometry::FaceGeometry>)
+output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+
+# Extracts face geometry for multiple faces from a vector of landmark lists.
+node {
+  calculator: "FaceGeometryPipelineCalculator"
+  input_side_packet: "ENVIRONMENT:environment"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
+  output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
+  options: {
+    [mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
+      metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
+    }
+  }
+}