Project import generated by Copybara.

GitOrigin-RevId: 65b427572550bd9c5bc5f053eeea0f44340d5673

WORKSPACE

@@ -35,8 +35,8 @@ http_archive(
 
 http_archive(
     name = "rules_cc",
-    strip_prefix = "rules_cc-master",
+    strip_prefix = "rules_cc-main",
-    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/main.zip"],
 )
 
 http_archive(

mediapipe/calculators/core/BUILD

@@ -419,6 +419,23 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "make_pair_calculator_test",
+    size = "small",
+    srcs = ["make_pair_calculator_test.cc"],
+    deps = [
+        ":make_pair_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:validate_type",
+        "//mediapipe/util:packet_test_util",
+        "//mediapipe/util:time_series_test_util",
+    ],
+)
+
 cc_library(
     name = "matrix_multiply_calculator",
     srcs = ["matrix_multiply_calculator.cc"],

mediapipe/calculators/core/make_pair_calculator_test.cc

@@ -0,0 +1,70 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/timestamp.h"
+#include "mediapipe/framework/tool/validate_type.h"
+#include "mediapipe/util/packet_test_util.h"
+#include "mediapipe/util/time_series_test_util.h"
+
+namespace mediapipe {
+
+class MakePairCalculatorTest
+    : public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
+ protected:
+  void SetUp() override {
+    calculator_name_ = "MakePairCalculator";
+    num_input_streams_ = 2;
+  }
+};
+
+TEST_F(MakePairCalculatorTest, ProducesExpectedPairs) {
+  InitializeGraph();
+  AppendInputPacket(new std::string("first packet"), Timestamp(1),
+                    /* input_index= */ 0);
+  AppendInputPacket(new std::string("second packet"), Timestamp(5),
+                    /* input_index= */ 0);
+  AppendInputPacket(new int(10), Timestamp(1), /* input_index= */ 1);
+  AppendInputPacket(new int(20), Timestamp(5), /* input_index= */ 1);
+
+  MP_ASSERT_OK(RunGraph());
+
+  EXPECT_THAT(
+      output().packets,
+      ::testing::ElementsAre(
+          mediapipe::PacketContainsTimestampAndPayload<
+              std::pair<Packet, Packet>>(
+              Timestamp(1),
+              ::testing::Pair(
+                  mediapipe::PacketContainsTimestampAndPayload<std::string>(
+                      Timestamp(1), std::string("first packet")),
+                  mediapipe::PacketContainsTimestampAndPayload<int>(
+                      Timestamp(1), 10))),
+          mediapipe::PacketContainsTimestampAndPayload<
+              std::pair<Packet, Packet>>(
+              Timestamp(5),
+              ::testing::Pair(
+                  mediapipe::PacketContainsTimestampAndPayload<std::string>(
+                      Timestamp(5), std::string("second packet")),
+                  mediapipe::PacketContainsTimestampAndPayload<int>(
+                      Timestamp(5), 20)))));
+}
+
+}  // namespace mediapipe

mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc

@@ -660,14 +660,18 @@ absl::Status ContentZoomingCalculator::Process(
   // Prevent box from extending beyond the image after camera smoothing.
   if (path_offset_y - ceil(path_height / 2.0) < 0) {
     path_offset_y = ceil(path_height / 2.0);
+    MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
   } else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
     path_offset_y = frame_height_ - ceil(path_height / 2.0);
+    MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
   }
 
   if (path_offset_x - ceil(path_width / 2.0) < 0) {
     path_offset_x = ceil(path_width / 2.0);
+    MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
   } else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {
     path_offset_x = frame_width_ - ceil(path_width / 2.0);
+    MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
   }
 
   // Convert to top/bottom borders to remove.

mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto

@@ -68,5 +68,10 @@ message FaceBoxAdjusterCalculatorOptions {
 
   // The max amount of time to use an old eye distance when the face look angle
   // is unstable.
-  optional int32 max_facesize_history_us = 9 [default = 8000000];
+  optional int32 max_facesize_history_us = 9 [default = 300000000];
+
+  // Scale factor of face width to shift based on pan look angle.
+  optional float pan_position_shift_scale = 15 [default = 0.5];
+  // Scale factor of face height to shift based on tilt look angle.
+  optional float tilt_position_shift_scale = 16 [default = 0.5];
 }

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc

@@ -209,6 +209,12 @@ absl::Status KinematicPathSolver::GetState(int* position) {
   return absl::OkStatus();
 }
 
+absl::Status KinematicPathSolver::SetState(const int position) {
+  RET_CHECK(initialized_) << "SetState called before first observation added.";
+  current_position_px_ = position;
+  return absl::OkStatus();
+}
+
 absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
   RET_CHECK(initialized_)
       << "GetTargetPosition called before first observation added.";

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h

@@ -48,6 +48,8 @@ class KinematicPathSolver {
   absl::Status UpdatePrediction(const int64 time_us);
   // Get the state at a time.
   absl::Status GetState(int* position);
+  // Overwrite the current state value.
+  absl::Status SetState(const int position);
   // Update PixelPerDegree value.
   absl::Status UpdatePixelsPerDegree(const float pixels_per_degree);
   // Provide the current target position of the reframe action.

mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc

@@ -371,6 +371,27 @@ TEST(KinematicPathSolverTest, TimestampSmoothing) {
   EXPECT_EQ(state, 701);
 }
 
+TEST(KinematicPathSolverTest, PassSetPosition) {
+  KinematicOptions options;
+  // Set min motion to 2deg
+  options.set_min_motion_to_reframe(1.0);
+  options.set_update_rate_seconds(.0000001);
+  options.set_max_update_rate(1.0);
+  options.set_max_velocity(18);
+  // Set degrees / pixel to 8.3
+  KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
+  int state;
+  MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
+  // Move target by 10px / 8.3 = 1.2deg
+  MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
+  MP_ASSERT_OK(solver.GetState(&state));
+  // Expect cam to move.
+  EXPECT_EQ(state, 410);
+  MP_ASSERT_OK(solver.SetState(400));
+  MP_ASSERT_OK(solver.GetState(&state));
+  EXPECT_EQ(state, 400);
+}
+
 }  // namespace
 }  // namespace autoflip
 }  // namespace mediapipe

mediapipe/java/com/google/mediapipe/framework/jni/BUILD

@@ -73,7 +73,6 @@ cc_library(
         ],
         "//mediapipe/gpu:disable_gpu": [],
     }),
-    features = ["-no_undefined"],
     linkopts = select({
         "//conditions:default": [],
         "//mediapipe:android": [

mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl

@@ -243,6 +243,8 @@ def _mediapipe_jni(name, gen_libmediapipe, calculators = []):
             "//mediapipe:android_arm64": ["@android_opencv//:libopencv_java3_so_arm64-v8a"],
             "//mediapipe:android_armeabi": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
             "//mediapipe:android_arm": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
+            "//mediapipe:android_x86": ["@android_opencv//:libopencv_java3_so_x86"],
+            "//mediapipe:android_x86_64": ["@android_opencv//:libopencv_java3_so_x86_64"],
             "//conditions:default": [],
         }),
         alwayslink = 1,

mediapipe/java/com/google/mediapipe/solutioncore/BUILD

@@ -75,6 +75,7 @@ cc_binary(
     # TODO: Add more calculators to support other top-level solutions.
     deps = [
         "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+        "//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu_image",
         "//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image",
     ],
 )

mediapipe/java/com/google/mediapipe/solutioncore/CameraInput.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.app.Activity;
 import com.google.mediapipe.components.CameraHelper;

mediapipe/java/com/google/mediapipe/solutioncore/ErrorListener.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 /** Interface for the customizable MediaPipe solution error listener. */
 public interface ErrorListener {

mediapipe/java/com/google/mediapipe/solutioncore/ImageSolutionBase.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.content.Context;
 import android.graphics.Bitmap;

mediapipe/java/com/google/mediapipe/solutioncore/ImageSolutionResult.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.graphics.Bitmap;
 import com.google.mediapipe.framework.AndroidPacketGetter;

mediapipe/java/com/google/mediapipe/solutioncore/OutputHandler.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.util.Log;
 import com.google.mediapipe.framework.MediaPipeException;

mediapipe/java/com/google/mediapipe/solutioncore/ResultGlRenderer.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 /** Interface for the customizable MediaPipe solution result OpenGL renderer. */
 public interface ResultGlRenderer<T extends ImageSolutionResult> {

mediapipe/java/com/google/mediapipe/solutioncore/ResultListener.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 /** Interface for the customizable MediaPipe solution result listener. */
 public interface ResultListener<T> {

mediapipe/java/com/google/mediapipe/solutioncore/SolutionBase.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;

mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceView.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.content.Context;
 import android.opengl.GLES20;

mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceViewRenderer.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import android.graphics.SurfaceTexture;
 import com.google.mediapipe.components.GlSurfaceViewRenderer;

mediapipe/java/com/google/mediapipe/solutioncore/SolutionInfo.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;

mediapipe/java/com/google/mediapipe/solutioncore/SolutionResult.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.mediapipe.solutionbase;
+package com.google.mediapipe.solutioncore;
 
 /**
  * Interface of the MediaPipe solution result. Any MediaPipe solution-specific result class should

mediapipe/java/com/google/mediapipe/solutioncore/proguard.cfg

@@ -0,0 +1,6 @@
+# Keep public members of our public interfaces. This also prevents the
+# obfuscation of the corresponding methods in classes implementing them,
+# such as implementations of PacketCallback#process.
+-keep public interface com.google.mediapipe.solutioncore.* {
+  public *;
+}

mediapipe/java/com/google/mediapipe/solutions/hands/BUILD

@@ -24,6 +24,7 @@ android_library(
     ],
     assets = [
         "//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image.binarypb",
+        "//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu_image.binarypb",
         "//mediapipe/modules/hand_landmark:handedness.txt",
         "//mediapipe/modules/hand_landmark:hand_landmark.tflite",
         "//mediapipe/modules/palm_detection:palm_detection.tflite",
@@ -36,7 +37,7 @@ android_library(
         "//mediapipe/framework/formats:classification_java_proto_lite",
         "//mediapipe/framework/formats:landmark_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",
-        "//mediapipe/java/com/google/mediapipe/solutionbase:solution_base",
+        "//mediapipe/java/com/google/mediapipe/solutioncore:solution_base",
         "//third_party:autovalue",
         "@maven//:androidx_annotation_annotation",
         "@maven//:com_google_code_findbugs_jsr305",

mediapipe/java/com/google/mediapipe/solutions/hands/Hands.java

@@ -21,11 +21,11 @@ import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
 import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
 import com.google.mediapipe.framework.MediaPipeException;
 import com.google.mediapipe.framework.Packet;
-import com.google.mediapipe.solutionbase.ErrorListener;
+import com.google.mediapipe.solutioncore.ErrorListener;
-import com.google.mediapipe.solutionbase.ImageSolutionBase;
+import com.google.mediapipe.solutioncore.ImageSolutionBase;
-import com.google.mediapipe.solutionbase.OutputHandler;
+import com.google.mediapipe.solutioncore.OutputHandler;
-import com.google.mediapipe.solutionbase.ResultListener;
+import com.google.mediapipe.solutioncore.ResultListener;
-import com.google.mediapipe.solutionbase.SolutionInfo;
+import com.google.mediapipe.solutioncore.SolutionInfo;
 import java.util.HashMap;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -41,7 +41,8 @@ public class Hands extends ImageSolutionBase {
   private static final String TAG = "Hands";
 
   private static final String NUM_HANDS = "num_hands";
-  private static final String SOLUTION_GRAPH_NAME = "hand_landmark_tracking_gpu_image.binarypb";
+  private static final String GPU_GRAPH_NAME = "hand_landmark_tracking_gpu_image.binarypb";
+  private static final String CPU_GRAPH_NAME = "hand_landmark_tracking_cpu_image.binarypb";
   private static final String IMAGE_INPUT_STREAM = "image";
   private static final ImmutableList<String> OUTPUT_STREAMS =
       ImmutableList.of("multi_hand_landmarks", "multi_handedness", "image");
@@ -82,7 +83,7 @@ public class Hands extends ImageSolutionBase {
 
     SolutionInfo solutionInfo =
         SolutionInfo.builder()
-            .setBinaryGraphPath(SOLUTION_GRAPH_NAME)
+            .setBinaryGraphPath(options.runOnGpu() ? GPU_GRAPH_NAME : CPU_GRAPH_NAME)
             .setImageInputStreamName(IMAGE_INPUT_STREAM)
             .setOutputStreamNames(OUTPUT_STREAMS)
             .setStaticImageMode(options.mode() == HandsOptions.STATIC_IMAGE_MODE)

mediapipe/java/com/google/mediapipe/solutions/hands/HandsOptions.java

@@ -33,6 +33,8 @@ import com.google.auto.value.AutoValue;
  * <p>minTrackingConfidence: Minimum confidence value ([0.0, 1.0]) for the hand landmarks to be
  * considered tracked successfully. See details in
  * https://solutions.mediapipe.dev/hands#min_tracking_confidence.
+ *
+ * <p>runOnGpu: Whether to run pipeline on GPU or CPU. Default to true.
  */
 @AutoValue
 public abstract class HandsOptions {
@@ -57,13 +59,22 @@ public abstract class HandsOptions {
 
   public abstract float minTrackingConfidence();
 
+  public abstract boolean runOnGpu();
+
   public static Builder builder() {
-    return new AutoValue_HandsOptions.Builder();
+    return new AutoValue_HandsOptions.Builder().withDefaultValues();
   }
 
   /** Builder for {@link HandsOptions}. */
   @AutoValue.Builder
   public abstract static class Builder {
+    public Builder withDefaultValues() {
+      return setMaxNumHands(2)
+          .setMinDetectionConfidence(0.5f)
+          .setMinTrackingConfidence(0.5f)
+          .setRunOnGpu(true);
+    }
+
     public abstract Builder setMode(int value);
 
     public abstract Builder setMaxNumHands(int value);
@@ -72,6 +83,8 @@ public abstract class HandsOptions {
 
     public abstract Builder setMinTrackingConfidence(float value);
 
+    public abstract Builder setRunOnGpu(boolean value);
+
     public abstract HandsOptions build();
   }
 }

mediapipe/java/com/google/mediapipe/solutions/hands/HandsResult.java

@@ -21,7 +21,7 @@ import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
 import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
 import com.google.mediapipe.framework.Packet;
 import com.google.mediapipe.framework.TextureFrame;
-import com.google.mediapipe.solutionbase.ImageSolutionResult;
+import com.google.mediapipe.solutioncore.ImageSolutionResult;
 import java.util.List;
 
 /**

mediapipe/python/solutions/hands_test.py

@@ -38,7 +38,7 @@ EXPECTED_HAND_COORDINATES_PREDICTION = [[[144, 345], [211, 323], [257, 286],
                                          [185, 19], [138, 208], [131, 127],
                                          [124, 77], [117, 36], [106, 222],
                                          [92, 159], [79, 124], [68, 93]],
-                                        [[577, 40], [504, 56], [459, 94],
+                                        [[577, 37], [504, 56], [459, 94],
                                          [429, 146], [397, 182], [496, 167],
                                          [479, 245], [469, 292], [464, 330],
                                          [540, 177], [534, 265], [533, 319],