diff --git a/WORKSPACE b/WORKSPACE index 9dbbff399..c2aaca658 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -350,6 +350,9 @@ maven_install( "com.google.auto.value:auto-value:1.8.1", "com.google.auto.value:auto-value-annotations:1.8.1", "com.google.code.findbugs:jsr305:latest.release", + "com.google.android.datatransport:transport-api:3.0.0", + "com.google.android.datatransport:transport-backend-cct:3.1.0", + "com.google.android.datatransport:transport-runtime:3.1.0", "com.google.flogger:flogger-system-backend:0.6", "com.google.flogger:flogger:0.6", "com.google.guava:guava:27.0.1-android", diff --git a/docs/getting_started/android_solutions.md b/docs/getting_started/android_solutions.md index 19d7403dc..9df98043f 100644 --- a/docs/getting_started/android_solutions.md +++ b/docs/getting_started/android_solutions.md @@ -36,16 +36,6 @@ dependencies { implementation 'com.google.mediapipe:facemesh:latest.release' // Optional: MediaPipe Hands Solution. implementation 'com.google.mediapipe:hands:latest.release' - // MediaPipe deps - implementation 'com.google.flogger:flogger:0.6' - implementation 'com.google.flogger:flogger-system-backend:0.6' - implementation 'com.google.guava:guava:27.0.1-android' - implementation 'com.google.protobuf:protobuf-java:3.11.4' - // CameraX core library - def camerax_version = "1.0.0-beta10" - implementation "androidx.camera:camera-core:$camerax_version" - implementation "androidx.camera:camera-camera2:$camerax_version" - implementation "androidx.camera:camera-lifecycle:$camerax_version" } ``` @@ -84,3 +74,58 @@ To build these apps: by default. If needed, for example to run the apps on Android Emulator, set the `RUN_ON_GPU` boolean variable to `false` in the app's `MainActivity.java` to run the pipeline and model inference on CPU. + +## MediaPipe Solution APIs Terms of Service + +Last modified: November 12, 2021 + +Use of MediaPipe Solution APIs is subject to the +[Google APIs Terms of Service](https://developers.google.com/terms), +[Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy), +and the terms below. Please check back from time to time as these terms and +policies are occasionally updated. + +**Privacy** + +When you use MediaPipe Solution APIs, processing of the input data (e.g. images, +video, text) fully happens on-device, and **MediaPipe does not send that input +data to Google servers**. As a result, you can use our APIs for processing data +that should not leave the device. + +MediaPipe Android Solution APIs will contact Google servers from time to time in +order to receive things like bug fixes, updated models, and hardware accelerator +compatibility information. MediaPipe Android Solution APIs also send metrics +about the performance and utilization of the APIs in your app to Google. Google +uses this metrics data to measure performance, API usage, debug, maintain and +improve the APIs, and detect misuse or abuse, as further described in our +[Privacy Policy](https://policies.google.com/privacy). + +**You are responsible for obtaining informed consent from your app users about +Google’s processing of MediaPipe metrics data as required by applicable law.** + +Data we collect may include the following, across all MediaPipe Android Solution +APIs: + +- Device information (such as manufacturer, model, OS version and build) and + available ML hardware accelerators (GPU and DSP). Used for diagnostics and + usage analytics. + +- App identification information (package name / bundle id, app version). Used + for diagnostics and usage analytics. + +- API configuration (such as image format, resolution, and MediaPipe version + used). Used for diagnostics and usage analytics. + +- Event type (such as initialize, download model, update, run, and detection). + Used for diagnostics and usage analytics. + +- Error codes. Used for diagnostics. + +- Performance metrics. Used for diagnostics. + +- Per-installation identifiers that do not uniquely identify a user or + physical device. Used for operation of remote configuration and usage + analytics. + +- Network request sender IP addresses. Used for remote configuration + diagnostics. Collected IP addresses are retained temporarily. diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md index d5423cde3..04d429987 100644 --- a/docs/solutions/face_detection.md +++ b/docs/solutions/face_detection.md @@ -218,14 +218,13 @@ camera.start(); ### Android Solution API Please first follow general -[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api) -to add MediaPipe Gradle dependencies, then try the Face Detection Solution API -in the companion -[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection) -following -[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio) +[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle +dependencies and try the Android Solution API in the companion +[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facedetection), and learn more in the usage example below. +Supported configuration options: + * [staticImageMode](#static_image_mode) * [modelSelection](#model_selection) diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md index fcc0b0c4c..57bf4de5b 100644 --- a/docs/solutions/face_mesh.md +++ b/docs/solutions/face_mesh.md @@ -487,12 +487,9 @@ camera.start(); ### Android Solution API Please first follow general -[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api) -to add MediaPipe Gradle dependencies, then try the Face Mesh Solution API in the -companion -[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh) -following -[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio) +[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle +dependencies and try the Android Solution API in the companion +[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/facemesh), and learn more in the usage example below. Supported configuration options: diff --git a/docs/solutions/hands.md b/docs/solutions/hands.md index 8c8d02622..e8a75ee16 100644 --- a/docs/solutions/hands.md +++ b/docs/solutions/hands.md @@ -398,13 +398,10 @@ camera.start(); ### Android Solution API Please first follow general -[instructions](../getting_started/android_solutions.md#integrate-mediapipe-android-solutions-api) -to add MediaPipe Gradle dependencies, then try the Hands Solution API in the -companion -[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands) -following -[these instructions](../getting_started/android_solutions.md#build-solution-example-apps-in-android-studio) -and learn more in usage example below. +[instructions](../getting_started/android_solutions.md) to add MediaPipe Gradle +dependencies and try the Android Solution API in the companion +[example Android Studio project](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/solutions/hands), +and learn more in the usage example below. Supported configuration options: diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 92ed084c9..6bd6cfe91 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -1169,6 +1169,7 @@ cc_library( "//mediapipe/framework:collection_item_id", "//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:status", + "//mediapipe/util:rectangle_util", "@com_google_absl//absl/memory", ], alwayslink = 1, diff --git a/mediapipe/calculators/util/association_calculator.h b/mediapipe/calculators/util/association_calculator.h index 6e5b480ce..037ea838c 100644 --- a/mediapipe/calculators/util/association_calculator.h +++ b/mediapipe/calculators/util/association_calculator.h @@ -26,20 +26,10 @@ #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/rectangle.h" #include "mediapipe/framework/port/status.h" +#include "mediapipe/util/rectangle_util.h" namespace mediapipe { -// Computes the overlap similarity based on Intersection over Union (IoU) of -// two rectangles. -inline float OverlapSimilarity(const Rectangle_f& rect1, - const Rectangle_f& rect2) { - if (!rect1.Intersects(rect2)) return 0.0f; - // Compute IoU similarity score. - const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area(); - const float normalization = rect1.Area() + rect2.Area() - intersection_area; - return normalization > 0.0f ? intersection_area / normalization : 0.0f; -} - // AssocationCalculator accepts multiple inputs of vectors of type T that can // be converted to Rectangle_f. The output is a vector of type T that contains // elements from the input vectors that don't overlap with each other. When @@ -187,7 +177,7 @@ class AssociationCalculator : public CalculatorBase { for (auto uit = current->begin(); uit != current->end();) { ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit)); - if (OverlapSimilarity(cur_rect, prev_rect) > + if (CalculateIou(cur_rect, prev_rect) > options_.min_similarity_threshold()) { std::pair prev_id = GetId(*uit); // If prev_id.first is false when some element doesn't have an ID, @@ -232,7 +222,7 @@ class AssociationCalculator : public CalculatorBase { } const Rectangle_f& prev_rect = get_prev_rectangle.value(); - if (OverlapSimilarity(cur_rect, prev_rect) > + if (CalculateIou(cur_rect, prev_rect) > options_.min_similarity_threshold()) { std::pair prev_id = GetId(prev_input_vec[ui]); // If prev_id.first is false when some element doesn't have an ID, diff --git a/mediapipe/examples/android/solutions/facedetection/build.gradle b/mediapipe/examples/android/solutions/facedetection/build.gradle index a3264a1b8..c3ebd94ac 100644 --- a/mediapipe/examples/android/solutions/facedetection/build.gradle +++ b/mediapipe/examples/android/solutions/facedetection/build.gradle @@ -35,17 +35,7 @@ dependencies { testImplementation 'junit:junit:4.+' androidTestImplementation 'androidx.test.ext:junit:1.1.2' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' - // MediaPipe Face Detection Solution components. + // MediaPipe Face Detection Solution. implementation 'com.google.mediapipe:solution-core:latest.release' implementation 'com.google.mediapipe:facedetection:latest.release' - // MediaPipe deps - implementation 'com.google.flogger:flogger:0.6' - implementation 'com.google.flogger:flogger-system-backend:0.6' - implementation 'com.google.guava:guava:27.0.1-android' - implementation 'com.google.protobuf:protobuf-java:3.11.4' - // CameraX core library - def camerax_version = "1.0.0-beta10" - implementation "androidx.camera:camera-core:$camerax_version" - implementation "androidx.camera:camera-camera2:$camerax_version" - implementation "androidx.camera:camera-lifecycle:$camerax_version" } diff --git a/mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml b/mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml index ffb743d2d..f150bd41a 100644 --- a/mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml +++ b/mediapipe/examples/android/solutions/facedetection/src/main/AndroidManifest.xml @@ -11,6 +11,9 @@ + + + + + + + + + InputSidePackets().HasTag(kStateCache)) { cc->InputSidePackets().Tag(kStateCache).Set(); } + if (cc->Outputs().HasTag(kCameraActive)) { + cc->Outputs().Tag(kCameraActive).Set(); + } return absl::OkStatus(); } @@ -649,6 +655,13 @@ absl::Status ContentZoomingCalculator::Process( path_solver_tilt_->ClearHistory(); path_solver_zoom_->ClearHistory(); } + const bool camera_active = + is_animating || pan_state || tilt_state || zoom_state; + if (cc->Outputs().HasTag(kCameraActive)) { + cc->Outputs() + .Tag(kCameraActive) + .AddPacket(MakePacket(camera_active).At(cc->InputTimestamp())); + } // Compute smoothed zoom camera path. MP_RETURN_IF_ERROR(path_solver_zoom_->AddObservation( diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index 83d28ceb3..5408a7add 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -75,6 +75,9 @@ class TagIndexMap { std::map>> map_; }; +class Graph; +class NodeBase; + // These structs are used internally to store information about the endpoints // of a connection. struct SourceBase; @@ -109,7 +112,7 @@ class MultiPort : public Single { // These classes wrap references to the underlying source/destination // endpoints, adding type information and the user-visible API. -template +template class DestinationImpl { public: using Base = DestinationBase; @@ -121,13 +124,12 @@ class DestinationImpl { }; template -class DestinationImpl - : public MultiPort> { +class MultiDestinationImpl : public MultiPort> { public: - using MultiPort>::MultiPort; + using MultiPort>::MultiPort; }; -template +template class SourceImpl { public: using Base = SourceBase; @@ -135,9 +137,9 @@ class SourceImpl { // Src is used as the return type of fluent methods below. Since these are // single-port methods, it is desirable to always decay to a reference to the // single-port superclass, even if they are called on a multiport. - using Src = SourceImpl; + using Src = SourceImpl; template - using Dst = DestinationImpl; + using Dst = DestinationImpl; // clang-format off template @@ -173,10 +175,9 @@ class SourceImpl { }; template -class SourceImpl - : public MultiPort> { +class MultiSourceImpl : public MultiPort> { public: - using MultiPort>::MultiPort; + using MultiPort>::MultiPort; }; // A source and a destination correspond to an output/input stream on a node, @@ -185,14 +186,23 @@ class SourceImpl // For graph inputs/outputs, however, the inputs are sources, and the outputs // are destinations. This is because graph ports are connected "from inside" // when building the graph. -template -using Source = SourceImpl; -template -using SideSource = SourceImpl; -template -using Destination = DestinationImpl; -template -using SideDestination = DestinationImpl; +template +using Source = SourceImpl; +template +using MultiSource = MultiSourceImpl; +template +using SideSource = SourceImpl; +template +using MultiSideSource = MultiSourceImpl; + +template +using Destination = DestinationImpl; +template +using SideDestination = DestinationImpl; +template +using MultiDestination = MultiDestinationImpl; +template +using MultiSideDestination = MultiDestinationImpl; class NodeBase { public: @@ -202,45 +212,67 @@ class NodeBase { // of its entries by index. However, for nodes without visible contracts we // can't know whether a tag is indexable or not, so we would need the // multi-port to also be usable as a port directly (representing index 0). - Source Out(const std::string& tag) { - return Source(&out_streams_[tag]); + MultiSource<> Out(const std::string& tag) { + return MultiSource<>(&out_streams_[tag]); } - Destination In(const std::string& tag) { - return Destination(&in_streams_[tag]); + MultiDestination<> In(const std::string& tag) { + return MultiDestination<>(&in_streams_[tag]); } - SideSource SideOut(const std::string& tag) { - return SideSource(&out_sides_[tag]); + MultiSideSource<> SideOut(const std::string& tag) { + return MultiSideSource<>(&out_sides_[tag]); } - SideDestination SideIn(const std::string& tag) { - return SideDestination(&in_sides_[tag]); + MultiSideDestination<> SideIn(const std::string& tag) { + return MultiSideDestination<>(&in_sides_[tag]); } template auto operator[](const PortCommon& port) { + using PayloadT = + typename PortCommon::PayloadT; if constexpr (std::is_same_v) { - return Source(&out_streams_[port.Tag()]); + auto* base = &out_streams_[port.Tag()]; + if constexpr (kIsMultiple) { + return MultiSource(base); + } else { + return Source(base); + } } else if constexpr (std::is_same_v) { - return Destination(&in_streams_[port.Tag()]); + auto* base = &in_streams_[port.Tag()]; + if constexpr (kIsMultiple) { + return MultiDestination(base); + } else { + return Destination(base); + } } else if constexpr (std::is_same_v) { - return SideSource(&out_sides_[port.Tag()]); + auto* base = &out_sides_[port.Tag()]; + if constexpr (kIsMultiple) { + return MultiSideSource(base); + } else { + return SideSource(base); + } } else if constexpr (std::is_same_v) { - return SideDestination(&in_sides_[port.Tag()]); + auto* base = &in_sides_[port.Tag()]; + if constexpr (kIsMultiple) { + return MultiSideDestination(base); + } else { + return SideDestination(base); + } } else { static_assert(dependent_false::value, "Type not supported."); } } // Convenience methods for accessing purely index-based ports. - Source Out(int index) { return Out("")[index]; } + Source<> Out(int index) { return Out("")[index]; } - Destination In(int index) { return In("")[index]; } + Destination<> In(int index) { return In("")[index]; } - SideSource SideOut(int index) { return SideOut("")[index]; } + SideSource<> SideOut(int index) { return SideOut("")[index]; } - SideDestination SideIn(int index) { return SideIn("")[index]; } + SideDestination<> SideIn(int index) { return SideIn("")[index]; } template T& GetOptions() { @@ -277,11 +309,6 @@ class Node : public NodeBase { using GenericNode = Node; -template