cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 65b427572550bd9c5bc5f053eeea0f44340d5673
This commit is contained in:
MediaPipe Team 2021-06-28 10:06:04 -07:00 committed by jqtang
parent 139237092f
commit 374f5e2e7e
29 changed files with 175 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
WORKSPACE
View File

@ -35,8 +35,8 @@ http_archive(
http_archive(
name = "rules_cc",
strip_prefix = "rules_cc-master",
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
strip_prefix = "rules_cc-main",
urls = ["https://github.com/bazelbuild/rules_cc/archive/main.zip"],
)
http_archive(

17
mediapipe/calculators/core/BUILD
View File

@ -419,6 +419,23 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "make_pair_calculator_test",
size = "small",
srcs = ["make_pair_calculator_test.cc"],
deps = [
":make_pair_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:validate_type",
"//mediapipe/util:packet_test_util",
"//mediapipe/util:time_series_test_util",
],
)
cc_library(
name = "matrix_multiply_calculator",
srcs = ["matrix_multiply_calculator.cc"],

70
mediapipe/calculators/core/make_pair_calculator_test.cc Normal file
View File

@ -0,0 +1,70 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/framework/tool/validate_type.h"
#include "mediapipe/util/packet_test_util.h"
#include "mediapipe/util/time_series_test_util.h"
namespace mediapipe {
class MakePairCalculatorTest
: public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
protected:
void SetUp() override {
calculator_name_ = "MakePairCalculator";
num_input_streams_ = 2;
}
};
TEST_F(MakePairCalculatorTest, ProducesExpectedPairs) {
InitializeGraph();
AppendInputPacket(new std::string("first packet"), Timestamp(1),
/* input_index= */ 0);
AppendInputPacket(new std::string("second packet"), Timestamp(5),
/* input_index= */ 0);
AppendInputPacket(new int(10), Timestamp(1), /* input_index= */ 1);
AppendInputPacket(new int(20), Timestamp(5), /* input_index= */ 1);
MP_ASSERT_OK(RunGraph());
EXPECT_THAT(
output().packets,
::testing::ElementsAre(
mediapipe::PacketContainsTimestampAndPayload<
std::pair<Packet, Packet>>(
Timestamp(1),
::testing::Pair(
mediapipe::PacketContainsTimestampAndPayload<std::string>(
Timestamp(1), std::string("first packet")),
mediapipe::PacketContainsTimestampAndPayload<int>(
Timestamp(1), 10))),
mediapipe::PacketContainsTimestampAndPayload<
std::pair<Packet, Packet>>(
Timestamp(5),
::testing::Pair(
mediapipe::PacketContainsTimestampAndPayload<std::string>(
Timestamp(5), std::string("second packet")),
mediapipe::PacketContainsTimestampAndPayload<int>(
Timestamp(5), 20)))));
}
} // namespace mediapipe

4
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
View File

@ -660,14 +660,18 @@ absl::Status ContentZoomingCalculator::Process(
// Prevent box from extending beyond the image after camera smoothing.
if (path_offset_y - ceil(path_height / 2.0) < 0) {
path_offset_y = ceil(path_height / 2.0);
MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
} else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) {
path_offset_y = frame_height_ - ceil(path_height / 2.0);
MP_RETURN_IF_ERROR(path_solver_tilt_->SetState(path_offset_y));
}
if (path_offset_x - ceil(path_width / 2.0) < 0) {
path_offset_x = ceil(path_width / 2.0);
MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
} else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) {
path_offset_x = frame_width_ - ceil(path_width / 2.0);
MP_RETURN_IF_ERROR(path_solver_pan_->SetState(path_offset_x));
}
// Convert to top/bottom borders to remove.

7
mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto
View File

@ -68,5 +68,10 @@ message FaceBoxAdjusterCalculatorOptions {
// The max amount of time to use an old eye distance when the face look angle
// is unstable.
optional int32 max_facesize_history_us = 9 [default = 8000000];
optional int32 max_facesize_history_us = 9 [default = 300000000];
// Scale factor of face width to shift based on pan look angle.
optional float pan_position_shift_scale = 15 [default = 0.5];
// Scale factor of face height to shift based on tilt look angle.
optional float tilt_position_shift_scale = 16 [default = 0.5];
}

6
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc
View File

@ -209,6 +209,12 @@ absl::Status KinematicPathSolver::GetState(int* position) {
return absl::OkStatus();
}
absl::Status KinematicPathSolver::SetState(const int position) {
RET_CHECK(initialized_) << "SetState called before first observation added.";
current_position_px_ = position;
return absl::OkStatus();
}
absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) {
RET_CHECK(initialized_)
<< "GetTargetPosition called before first observation added.";

2
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h
View File

@ -48,6 +48,8 @@ class KinematicPathSolver {
absl::Status UpdatePrediction(const int64 time_us);
// Get the state at a time.
absl::Status GetState(int* position);
// Overwrite the current state value.
absl::Status SetState(const int position);
// Update PixelPerDegree value.
absl::Status UpdatePixelsPerDegree(const float pixels_per_degree);
// Provide the current target position of the reframe action.

21
mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc
View File

@ -371,6 +371,27 @@ TEST(KinematicPathSolverTest, TimestampSmoothing) {
EXPECT_EQ(state, 701);
}
TEST(KinematicPathSolverTest, PassSetPosition) {
KinematicOptions options;
// Set min motion to 2deg
options.set_min_motion_to_reframe(1.0);
options.set_update_rate_seconds(.0000001);
options.set_max_update_rate(1.0);
options.set_max_velocity(18);
// Set degrees / pixel to 8.3
KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView);
int state;
MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0));
// Move target by 10px / 8.3 = 1.2deg
MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1));
MP_ASSERT_OK(solver.GetState(&state));
// Expect cam to move.
EXPECT_EQ(state, 410);
MP_ASSERT_OK(solver.SetState(400));
MP_ASSERT_OK(solver.GetState(&state));
EXPECT_EQ(state, 400);
}
} // namespace
} // namespace autoflip
} // namespace mediapipe

1
mediapipe/java/com/google/mediapipe/framework/jni/BUILD
View File

@ -73,7 +73,6 @@ cc_library(
],
"//mediapipe/gpu:disable_gpu": [],
}),
features = ["-no_undefined"],
linkopts = select({
"//conditions:default": [],
"//mediapipe:android": [

2
mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl
View File

@ -243,6 +243,8 @@ def _mediapipe_jni(name, gen_libmediapipe, calculators = []):
"//mediapipe:android_arm64": ["@android_opencv//:libopencv_java3_so_arm64-v8a"],
"//mediapipe:android_armeabi": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
"//mediapipe:android_arm": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"],
"//mediapipe:android_x86": ["@android_opencv//:libopencv_java3_so_x86"],
"//mediapipe:android_x86_64": ["@android_opencv//:libopencv_java3_so_x86_64"],
"//conditions:default": [],
}),
alwayslink = 1,

1
mediapipe/java/com/google/mediapipe/solutionbase/BUILD → mediapipe/java/com/google/mediapipe/solutioncore/BUILD
View File

@ -75,6 +75,7 @@ cc_binary(
# TODO: Add more calculators to support other top-level solutions.
deps = [
"//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu_image",
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image",
],
)

2
mediapipe/java/com/google/mediapipe/solutionbase/CameraInput.java → mediapipe/java/com/google/mediapipe/solutioncore/CameraInput.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.app.Activity;
import com.google.mediapipe.components.CameraHelper;

2
mediapipe/java/com/google/mediapipe/solutionbase/ErrorListener.java → mediapipe/java/com/google/mediapipe/solutioncore/ErrorListener.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
/** Interface for the customizable MediaPipe solution error listener. */
public interface ErrorListener {

2
mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionBase.java → mediapipe/java/com/google/mediapipe/solutioncore/ImageSolutionBase.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.content.Context;
import android.graphics.Bitmap;

2
mediapipe/java/com/google/mediapipe/solutionbase/ImageSolutionResult.java → mediapipe/java/com/google/mediapipe/solutioncore/ImageSolutionResult.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.graphics.Bitmap;
import com.google.mediapipe.framework.AndroidPacketGetter;

2
mediapipe/java/com/google/mediapipe/solutionbase/OutputHandler.java → mediapipe/java/com/google/mediapipe/solutioncore/OutputHandler.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.util.Log;
import com.google.mediapipe.framework.MediaPipeException;

2
mediapipe/java/com/google/mediapipe/solutionbase/ResultGlRenderer.java → mediapipe/java/com/google/mediapipe/solutioncore/ResultGlRenderer.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
/** Interface for the customizable MediaPipe solution result OpenGL renderer. */
public interface ResultGlRenderer<T extends ImageSolutionResult> {

2
mediapipe/java/com/google/mediapipe/solutionbase/ResultListener.java → mediapipe/java/com/google/mediapipe/solutioncore/ResultListener.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
/** Interface for the customizable MediaPipe solution result listener. */
public interface ResultListener<T> {

2
mediapipe/java/com/google/mediapipe/solutionbase/SolutionBase.java → mediapipe/java/com/google/mediapipe/solutioncore/SolutionBase.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;

2
mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceView.java → mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceView.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.content.Context;
import android.opengl.GLES20;

2
mediapipe/java/com/google/mediapipe/solutionbase/SolutionGlSurfaceViewRenderer.java → mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceViewRenderer.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import android.graphics.SurfaceTexture;
import com.google.mediapipe.components.GlSurfaceViewRenderer;

2
mediapipe/java/com/google/mediapipe/solutionbase/SolutionInfo.java → mediapipe/java/com/google/mediapipe/solutioncore/SolutionInfo.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;

2
mediapipe/java/com/google/mediapipe/solutionbase/SolutionResult.java → mediapipe/java/com/google/mediapipe/solutioncore/SolutionResult.java
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.solutionbase;
package com.google.mediapipe.solutioncore;
/**
* Interface of the MediaPipe solution result. Any MediaPipe solution-specific result class should

6
mediapipe/java/com/google/mediapipe/solutioncore/proguard.cfg Normal file
View File

@ -0,0 +1,6 @@
# Keep public members of our public interfaces. This also prevents the
# obfuscation of the corresponding methods in classes implementing them,
# such as implementations of PacketCallback#process.
-keep public interface com.google.mediapipe.solutioncore.* {
public *;
}

3
mediapipe/java/com/google/mediapipe/solutions/hands/BUILD
View File

@ -24,6 +24,7 @@ android_library(
],
assets = [
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_gpu_image.binarypb",
"//mediapipe/modules/hand_landmark:hand_landmark_tracking_cpu_image.binarypb",
"//mediapipe/modules/hand_landmark:handedness.txt",
"//mediapipe/modules/hand_landmark:hand_landmark.tflite",
"//mediapipe/modules/palm_detection:palm_detection.tflite",
@ -36,7 +37,7 @@ android_library(
"//mediapipe/framework/formats:classification_java_proto_lite",
"//mediapipe/framework/formats:landmark_java_proto_lite",
"//mediapipe/java/com/google/mediapipe/framework:android_framework",
"//mediapipe/java/com/google/mediapipe/solutionbase:solution_base",
"//mediapipe/java/com/google/mediapipe/solutioncore:solution_base",
"//third_party:autovalue",
"@maven//:androidx_annotation_annotation",
"@maven//:com_google_code_findbugs_jsr305",

15
mediapipe/java/com/google/mediapipe/solutions/hands/Hands.java
View File

@ -21,11 +21,11 @@ import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.solutionbase.ErrorListener;
import com.google.mediapipe.solutionbase.ImageSolutionBase;
import com.google.mediapipe.solutionbase.OutputHandler;
import com.google.mediapipe.solutionbase.ResultListener;
import com.google.mediapipe.solutionbase.SolutionInfo;
import com.google.mediapipe.solutioncore.ErrorListener;
import com.google.mediapipe.solutioncore.ImageSolutionBase;
import com.google.mediapipe.solutioncore.OutputHandler;
import com.google.mediapipe.solutioncore.ResultListener;
import com.google.mediapipe.solutioncore.SolutionInfo;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
@ -41,7 +41,8 @@ public class Hands extends ImageSolutionBase {
private static final String TAG = "Hands";
private static final String NUM_HANDS = "num_hands";
private static final String SOLUTION_GRAPH_NAME = "hand_landmark_tracking_gpu_image.binarypb";
private static final String GPU_GRAPH_NAME = "hand_landmark_tracking_gpu_image.binarypb";
private static final String CPU_GRAPH_NAME = "hand_landmark_tracking_cpu_image.binarypb";
private static final String IMAGE_INPUT_STREAM = "image";
private static final ImmutableList<String> OUTPUT_STREAMS =
ImmutableList.of("multi_hand_landmarks", "multi_handedness", "image");
@ -82,7 +83,7 @@ public class Hands extends ImageSolutionBase {
SolutionInfo solutionInfo =
SolutionInfo.builder()
.setBinaryGraphPath(SOLUTION_GRAPH_NAME)
.setBinaryGraphPath(options.runOnGpu() ? GPU_GRAPH_NAME : CPU_GRAPH_NAME)
.setImageInputStreamName(IMAGE_INPUT_STREAM)
.setOutputStreamNames(OUTPUT_STREAMS)
.setStaticImageMode(options.mode() == HandsOptions.STATIC_IMAGE_MODE)

15
mediapipe/java/com/google/mediapipe/solutions/hands/HandsOptions.java
View File

@ -33,6 +33,8 @@ import com.google.auto.value.AutoValue;
* <p>minTrackingConfidence: Minimum confidence value ([0.0, 1.0]) for the hand landmarks to be
* considered tracked successfully. See details in
* https://solutions.mediapipe.dev/hands#min_tracking_confidence.
*
* <p>runOnGpu: Whether to run pipeline on GPU or CPU. Default to true.
*/
@AutoValue
public abstract class HandsOptions {
@ -57,13 +59,22 @@ public abstract class HandsOptions {
public abstract float minTrackingConfidence();
public abstract boolean runOnGpu();
public static Builder builder() {
return new AutoValue_HandsOptions.Builder();
return new AutoValue_HandsOptions.Builder().withDefaultValues();
}
/** Builder for {@link HandsOptions}. */
@AutoValue.Builder
public abstract static class Builder {
public Builder withDefaultValues() {
return setMaxNumHands(2)
.setMinDetectionConfidence(0.5f)
.setMinTrackingConfidence(0.5f)
.setRunOnGpu(true);
}
public abstract Builder setMode(int value);
public abstract Builder setMaxNumHands(int value);
@ -72,6 +83,8 @@ public abstract class HandsOptions {
public abstract Builder setMinTrackingConfidence(float value);
public abstract Builder setRunOnGpu(boolean value);
public abstract HandsOptions build();
}
}

2
mediapipe/java/com/google/mediapipe/solutions/hands/HandsResult.java
View File

@ -21,7 +21,7 @@ import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.solutionbase.ImageSolutionResult;
import com.google.mediapipe.solutioncore.ImageSolutionResult;
import java.util.List;
/**

2
mediapipe/python/solutions/hands_test.py
View File

@ -38,7 +38,7 @@ EXPECTED_HAND_COORDINATES_PREDICTION = [[[144, 345], [211, 323], [257, 286],
[185, 19], [138, 208], [131, 127],
[124, 77], [117, 36], [106, 222],
[92, 159], [79, 124], [68, 93]],
[[577, 40], [504, 56], [459, 94],
[[577, 37], [504, 56], [459, 94],
[429, 146], [397, 182], [496, 167],
[479, 245], [469, 292], [464, 330],
[540, 177], [534, 265], [533, 319],