diff --git a/README.md b/README.md index 8cd42fa7b..2d9550d37 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Hair Segmentation [Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅ [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | | [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | | -[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | | +[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ | [KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | | [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | | [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | | @@ -79,6 +79,13 @@ run code search using ## Publications +* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html) + in Google Developers Blog +* [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html) + in Google Developers Blog +* [SignAll SDK: Sign language interface using MediaPipe is now available for + developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html) + in Google Developers Blog * [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html) in Google AI Blog * [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html) diff --git a/WORKSPACE b/WORKSPACE index a1dcb4724..7aa985d62 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -53,19 +53,12 @@ rules_foreign_cc_dependencies() all_content = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" # GoogleTest/GoogleMock framework. Used by most unit-tests. -# Last updated 2020-06-30. +# Last updated 2021-07-02. http_archive( name = "com_google_googletest", - urls = ["https://github.com/google/googletest/archive/aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e.zip"], - patches = [ - # fix for https://github.com/google/googletest/issues/2817 - "@//third_party:com_google_googletest_9d580ea80592189e6d44fa35bcf9cdea8bf620d6.diff" - ], - patch_args = [ - "-p1", - ], - strip_prefix = "googletest-aee0f9d9b5b87796ee8a0ab26b7587ec30e8858e", - sha256 = "04a1751f94244307cebe695a69cc945f9387a80b0ef1af21394a490697c5c895", + urls = ["https://github.com/google/googletest/archive/4ec4cd23f486bf70efcc5d2caa40f24368f752e3.zip"], + strip_prefix = "googletest-4ec4cd23f486bf70efcc5d2caa40f24368f752e3", + sha256 = "de682ea824bfffba05b4e33b67431c247397d6175962534305136aa06f92e049", ) # Google Benchmark library. @@ -353,9 +346,9 @@ maven_install( "com.google.android.material:material:aar:1.0.0-rc01", "com.google.auto.value:auto-value:1.8.1", "com.google.auto.value:auto-value-annotations:1.8.1", - "com.google.code.findbugs:jsr305:3.0.2", - "com.google.flogger:flogger-system-backend:0.3.1", - "com.google.flogger:flogger:0.3.1", + "com.google.code.findbugs:jsr305:latest.release", + "com.google.flogger:flogger-system-backend:latest.release", + "com.google.flogger:flogger:latest.release", "com.google.guava:guava:27.0.1-android", "com.google.guava:listenablefuture:1.0", "junit:junit:4.12", diff --git a/docs/getting_started/android_archive_library.md b/docs/getting_started/android_archive_library.md index ec34a8352..bb7125243 100644 --- a/docs/getting_started/android_archive_library.md +++ b/docs/getting_started/android_archive_library.md @@ -113,9 +113,9 @@ each project. androidTestImplementation 'androidx.test.ext:junit:1.1.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1' // MediaPipe deps - implementation 'com.google.flogger:flogger:0.3.1' - implementation 'com.google.flogger:flogger-system-backend:0.3.1' - implementation 'com.google.code.findbugs:jsr305:3.0.2' + implementation 'com.google.flogger:flogger:latest.release' + implementation 'com.google.flogger:flogger-system-backend:latest.release' + implementation 'com.google.code.findbugs:jsr305:latest.release' implementation 'com.google.guava:guava:27.0.1-android' implementation 'com.google.protobuf:protobuf-java:3.11.4' // CameraX core library diff --git a/docs/getting_started/hello_world_android.md b/docs/getting_started/hello_world_android.md index 9f277f799..6674d4023 100644 --- a/docs/getting_started/hello_world_android.md +++ b/docs/getting_started/hello_world_android.md @@ -31,8 +31,8 @@ stream on an Android device. ## Setup -1. Install MediaPipe on your system, see [MediaPipe installation guide] for - details. +1. Install MediaPipe on your system, see + [MediaPipe installation guide](./install.md) for details. 2. Install Android Development SDK and Android NDK. See how to do so also in [MediaPipe installation guide]. 3. Enable [developer options] on your Android device. @@ -770,7 +770,6 @@ If you ran into any issues, please see the full code of the tutorial [`ExternalTextureConverter`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java [`FrameLayout`]:https://developer.android.com/reference/android/widget/FrameLayout [`FrameProcessor`]:https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java -[MediaPipe installation guide]:./install.md [`PermissionHelper`]: https://github.com/google/mediapipe/tree/master/mediapipe/java/com/google/mediapipe/components/PermissionHelper.java [`SurfaceHolder.Callback`]:https://developer.android.com/reference/android/view/SurfaceHolder.Callback.html [`SurfaceView`]:https://developer.android.com/reference/android/view/SurfaceView diff --git a/docs/getting_started/hello_world_ios.md b/docs/getting_started/hello_world_ios.md index 06d79c67d..9f9e7c979 100644 --- a/docs/getting_started/hello_world_ios.md +++ b/docs/getting_started/hello_world_ios.md @@ -31,8 +31,8 @@ stream on an iOS device. ## Setup -1. Install MediaPipe on your system, see [MediaPipe installation guide] for - details. +1. Install MediaPipe on your system, see + [MediaPipe installation guide](./install.md) for details. 2. Setup your iOS device for development. 3. Setup [Bazel] on your system to build and deploy the iOS app. @@ -560,6 +560,5 @@ appropriate `BUILD` file dependencies for the edge detection graph. [Bazel]:https://bazel.build/ [`edge_detection_mobile_gpu.pbtxt`]:https://github.com/google/mediapipe/tree/master/mediapipe/graphs/edge_detection/edge_detection_mobile_gpu.pbtxt -[MediaPipe installation guide]:./install.md [common]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/common) [helloworld]:(https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/helloworld) diff --git a/docs/getting_started/javascript.md b/docs/getting_started/javascript.md index 98a4f19bc..5cae5cbd4 100644 --- a/docs/getting_started/javascript.md +++ b/docs/getting_started/javascript.md @@ -22,6 +22,7 @@ Solution | NPM Package | Example [Face Detection][Fd-pg] | [@mediapipe/face_detection][Fd-npm] | [mediapipe.dev/demo/face_detection][Fd-demo] [Hands][H-pg] | [@mediapipe/hands][H-npm] | [mediapipe.dev/demo/hands][H-demo] [Holistic][Ho-pg] | [@mediapipe/holistic][Ho-npm] | [mediapipe.dev/demo/holistic][Ho-demo] +[Objectron][Ob-pg] | [@mediapipe/objectron][Ob-npm] | [mediapipe.dev/demo/objectron][Ob-demo] [Pose][P-pg] | [@mediapipe/pose][P-npm] | [mediapipe.dev/demo/pose][P-demo] [Selfie Segmentation][S-pg] | [@mediapipe/selfie_segmentation][S-npm] | [mediapipe.dev/demo/selfie_segmentation][S-demo] @@ -67,33 +68,24 @@ affecting your work, restrict your request to a `` number. e.g., [F-pg]: ../solutions/face_mesh#javascript-solution-api [Fd-pg]: ../solutions/face_detection#javascript-solution-api [H-pg]: ../solutions/hands#javascript-solution-api +[Ob-pg]: ../solutions/objectron#javascript-solution-api [P-pg]: ../solutions/pose#javascript-solution-api [S-pg]: ../solutions/selfie_segmentation#javascript-solution-api [Ho-npm]: https://www.npmjs.com/package/@mediapipe/holistic [F-npm]: https://www.npmjs.com/package/@mediapipe/face_mesh [Fd-npm]: https://www.npmjs.com/package/@mediapipe/face_detection [H-npm]: https://www.npmjs.com/package/@mediapipe/hands +[Ob-npm]: https://www.npmjs.com/package/@mediapipe/objectron [P-npm]: https://www.npmjs.com/package/@mediapipe/pose [S-npm]: https://www.npmjs.com/package/@mediapipe/selfie_segmentation [draw-npm]: https://www.npmjs.com/package/@mediapipe/drawing_utils [cam-npm]: https://www.npmjs.com/package/@mediapipe/camera_utils [ctrl-npm]: https://www.npmjs.com/package/@mediapipe/control_utils -[Ho-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/holistic -[F-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_mesh -[Fd-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/face_detection -[H-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/hands -[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/pose -[P-jsd]: https://www.jsdelivr.com/package/npm/@mediapipe/selfie_segmentation -[Ho-pen]: https://code.mediapipe.dev/codepen/holistic -[F-pen]: https://code.mediapipe.dev/codepen/face_mesh -[Fd-pen]: https://code.mediapipe.dev/codepen/face_detection -[H-pen]: https://code.mediapipe.dev/codepen/hands -[P-pen]: https://code.mediapipe.dev/codepen/pose -[S-pen]: https://code.mediapipe.dev/codepen/selfie_segmentation [Ho-demo]: https://mediapipe.dev/demo/holistic [F-demo]: https://mediapipe.dev/demo/face_mesh [Fd-demo]: https://mediapipe.dev/demo/face_detection [H-demo]: https://mediapipe.dev/demo/hands +[Ob-demo]: https://mediapipe.dev/demo/objectron [P-demo]: https://mediapipe.dev/demo/pose [S-demo]: https://mediapipe.dev/demo/selfie_segmentation [npm]: https://www.npmjs.com/package/@mediapipe diff --git a/docs/getting_started/python_framework.md b/docs/getting_started/python_framework.md index ece14bc91..688285d87 100644 --- a/docs/getting_started/python_framework.md +++ b/docs/getting_started/python_framework.md @@ -74,7 +74,7 @@ Mapping\[str, Packet\] | std::map | create_st np.ndarray
(cv.mat and PIL.Image) | mp::ImageFrame | create_image_frame(
        format=ImageFormat.SRGB,
        data=mat) | get_image_frame(packet) np.ndarray | mp::Matrix | create_matrix(data) | get_matrix(packet) Google Proto Message | Google Proto Message | create_proto(proto) | get_proto(packet) -List\[Proto\] | std::vector\ | create_proto_vector(proto_list) | get_proto_list(packet) +List\[Proto\] | std::vector\ | n/a | get_proto_list(packet) It's not uncommon that users create custom C++ classes and and send those into the graphs and calculators. To allow the custom classes to be used in Python diff --git a/docs/index.md b/docs/index.md index d3a22bd22..86d6ddc5e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,7 +45,7 @@ Hair Segmentation [Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅ [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | | [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | | -[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | | +[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ | [KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | | [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | | [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | | @@ -79,6 +79,13 @@ run code search using ## Publications +* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html) + in Google Developers Blog +* [Prosthesis control via Mirru App using MediaPipe hand tracking](https://developers.googleblog.com/2021/05/control-your-mirru-prosthesis-with-mediapipe-hand-tracking.html) + in Google Developers Blog +* [SignAll SDK: Sign language interface using MediaPipe is now available for + developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html) + in Google Developers Blog * [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html) in Google AI Blog * [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html) diff --git a/docs/solutions/objectron.md b/docs/solutions/objectron.md index 20dc3cace..457890372 100644 --- a/docs/solutions/objectron.md +++ b/docs/solutions/objectron.md @@ -224,29 +224,33 @@ where object detection simply runs on every image. Default to `0.99`. #### model_name -Name of the model to use for predicting 3D bounding box landmarks. Currently supports -`{'Shoe', 'Chair', 'Cup', 'Camera'}`. +Name of the model to use for predicting 3D bounding box landmarks. Currently +supports `{'Shoe', 'Chair', 'Cup', 'Camera'}`. Default to `Shoe`. #### focal_length -Camera focal length `(fx, fy)`, by default is defined in -[NDC space](#ndc-space). To use focal length `(fx_pixel, fy_pixel)` in -[pixel space](#pixel-space), users should provide `image_size` = `(image_width, -image_height)` to enable conversions inside the API. For further details about -NDC and pixel space, please see [Coordinate Systems](#coordinate-systems). +By default, camera focal length defined in [NDC space](#ndc-space), i.e., `(fx, +fy)`. Default to `(1.0, 1.0)`. To specify focal length in +[pixel space](#pixel-space) instead, i.e., `(fx_pixel, fy_pixel)`, users should +provide [`image_size`](#image_size) = `(image_width, image_height)` to enable +conversions inside the API. For further details about NDC and pixel space, +please see [Coordinate Systems](#coordinate-systems). #### principal_point -Camera principal point `(px, py)`, by default is defined in -[NDC space](#ndc-space). To use principal point `(px_pixel, py_pixel)` in -[pixel space](#pixel-space), users should provide `image_size` = `(image_width, -image_height)` to enable conversions inside the API. For further details about -NDC and pixel space, please see [Coordinate Systems](#coordinate-systems). +By default, camera principal point defined in [NDC space](#ndc-space), i.e., +`(px, py)`. Default to `(0.0, 0.0)`. To specify principal point in +[pixel space](#pixel-space), i.e.,`(px_pixel, py_pixel)`, users should provide +[`image_size`](#image_size) = `(image_width, image_height)` to enable +conversions inside the API. For further details about NDC and pixel space, +please see [Coordinate Systems](#coordinate-systems). #### image_size -(**Optional**) size `(image_width, image_height)` of the input image, **ONLY** -needed when use `focal_length` and `principal_point` in pixel space. +**Specify only when [`focal_length`](#focal_length) and +[`principal_point`](#principal_point) are specified in pixel space.** + +Size of the input image, i.e., `(image_width, image_height)`. ### Output @@ -356,6 +360,89 @@ with mp_objectron.Objectron(static_image_mode=False, cap.release() ``` +## JavaScript Solution API + +Please first see general [introduction](../getting_started/javascript.md) on +MediaPipe in JavaScript, then learn more in the companion [web demo](#resources) +and the following usage example. + +Supported configuration options: + +* [staticImageMode](#static_image_mode) +* [maxNumObjects](#max_num_objects) +* [minDetectionConfidence](#min_detection_confidence) +* [minTrackingConfidence](#min_tracking_confidence) +* [modelName](#model_name) +* [focalLength](#focal_length) +* [principalPoint](#principal_point) +* [imageSize](#image_size) + +```html + + + + + + + + + + + + +
+ + +
+ + +``` + +```javascript + +``` + ## Example Apps Please first see general instructions for @@ -561,11 +648,15 @@ py = -py_pixel * 2.0 / image_height + 1.0 [Announcing the Objectron Dataset](https://ai.googleblog.com/2020/11/announcing-objectron-dataset.html) * Google AI Blog: [Real-Time 3D Object Detection on Mobile Devices with MediaPipe](https://ai.googleblog.com/2020/03/real-time-3d-object-detection-on-mobile.html) -* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in CVPR 2021 +* Paper: [Objectron: A Large Scale Dataset of Object-Centric Videos in the + Wild with Pose Annotations](https://arxiv.org/abs/2012.09988), to appear in + CVPR 2021 * Paper: [MobilePose: Real-Time Pose Estimation for Unseen Objects with Weak Shape Supervision](https://arxiv.org/abs/2003.03522) * Paper: [Instant 3D Object Tracking with Applications in Augmented Reality](https://drive.google.com/open?id=1O_zHmlgXIzAdKljp20U_JUkEHOGG52R8) - ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth Workshop on Computer Vision for AR/VR, CVPR 2020 + ([presentation](https://www.youtube.com/watch?v=9ndF1AIo7h0)), Fourth + Workshop on Computer Vision for AR/VR, CVPR 2020 * [Models and model cards](./models.md#objectron) +* [Web demo](https://code.mediapipe.dev/codepen/objectron) * [Python Colab](https://mediapipe.page.link/objectron_py_colab) diff --git a/docs/solutions/selfie_segmentation.md b/docs/solutions/selfie_segmentation.md index f649bee72..fc063d1e7 100644 --- a/docs/solutions/selfie_segmentation.md +++ b/docs/solutions/selfie_segmentation.md @@ -96,6 +96,7 @@ Supported configuration options: ```python import cv2 import mediapipe as mp +import numpy as np mp_drawing = mp.solutions.drawing_utils mp_selfie_segmentation = mp.solutions.selfie_segmentation diff --git a/docs/solutions/solutions.md b/docs/solutions/solutions.md index 98bafe30e..5fffd5e4b 100644 --- a/docs/solutions/solutions.md +++ b/docs/solutions/solutions.md @@ -29,7 +29,7 @@ has_toc: false [Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅ [Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | | [Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | | -[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | | +[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ | [KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | | [AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | | [MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | | diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index f1d5805ef..be5a0aaf1 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -140,6 +140,16 @@ mediapipe_proto_library( ], ) +mediapipe_proto_library( + name = "graph_profile_calculator_proto", + srcs = ["graph_profile_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], +) + cc_library( name = "add_header_calculator", srcs = ["add_header_calculator.cc"], @@ -1200,3 +1210,45 @@ cc_test( "@com_google_absl//absl/strings", ], ) + +cc_library( + name = "graph_profile_calculator", + srcs = ["graph_profile_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":graph_profile_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_profile_cc_proto", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:packet", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + +cc_test( + name = "graph_profile_calculator_test", + srcs = ["graph_profile_calculator_test.cc"], + deps = [ + ":graph_profile_calculator", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_profile_cc_proto", + "//mediapipe/framework:test_calculators", + "//mediapipe/framework/deps:clock", + "//mediapipe/framework/deps:message_matchers", + "//mediapipe/framework/port:core_proto", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:threadpool", + "//mediapipe/framework/tool:simulation_clock_executor", + "//mediapipe/framework/tool:sink", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + ], +) diff --git a/mediapipe/calculators/core/graph_profile_calculator.cc b/mediapipe/calculators/core/graph_profile_calculator.cc new file mode 100644 index 000000000..9b9aa3bb7 --- /dev/null +++ b/mediapipe/calculators/core/graph_profile_calculator.cc @@ -0,0 +1,70 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "mediapipe/calculators/core/graph_profile_calculator.pb.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/packet.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_profile.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { +namespace api2 { + +// This calculator periodically copies the GraphProfile from +// mediapipe::GraphProfiler::CaptureProfile to the "PROFILE" output stream. +// +// Example config: +// node { +// calculator: "GraphProfileCalculator" +// output_stream: "FRAME:any_frame" +// output_stream: "PROFILE:graph_profile" +// } +// +class GraphProfileCalculator : public Node { + public: + static constexpr Input::Multiple kFrameIn{"FRAME"}; + static constexpr Output kProfileOut{"PROFILE"}; + + MEDIAPIPE_NODE_CONTRACT(kFrameIn, kProfileOut); + + static absl::Status UpdateContract(CalculatorContract* cc) { + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + auto options = cc->Options<::mediapipe::GraphProfileCalculatorOptions>(); + + if (prev_profile_ts_ == Timestamp::Unset() || + cc->InputTimestamp() - prev_profile_ts_ >= options.profile_interval()) { + prev_profile_ts_ = cc->InputTimestamp(); + GraphProfile result; + MP_RETURN_IF_ERROR(cc->GetProfilingContext()->CaptureProfile(&result)); + kProfileOut(cc).Send(result); + } + return absl::OkStatus(); + } + + private: + Timestamp prev_profile_ts_; +}; + +MEDIAPIPE_REGISTER_NODE(GraphProfileCalculator); + +} // namespace api2 +} // namespace mediapipe diff --git a/mediapipe/calculators/core/graph_profile_calculator.proto b/mediapipe/calculators/core/graph_profile_calculator.proto new file mode 100644 index 000000000..2bcc480c8 --- /dev/null +++ b/mediapipe/calculators/core/graph_profile_calculator.proto @@ -0,0 +1,30 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +option objc_class_prefix = "MediaPipe"; + +message GraphProfileCalculatorOptions { + extend mediapipe.CalculatorOptions { + optional GraphProfileCalculatorOptions ext = 367481815; + } + + // The interval in microseconds between successive reported GraphProfiles. + optional int64 profile_interval = 1 [default = 1000000]; +} diff --git a/mediapipe/calculators/core/graph_profile_calculator_test.cc b/mediapipe/calculators/core/graph_profile_calculator_test.cc new file mode 100644 index 000000000..44e914912 --- /dev/null +++ b/mediapipe/calculators/core/graph_profile_calculator_test.cc @@ -0,0 +1,207 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_profile.pb.h" +#include "mediapipe/framework/deps/clock.h" +#include "mediapipe/framework/deps/message_matchers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/proto_ns.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/port/threadpool.h" +#include "mediapipe/framework/tool/simulation_clock_executor.h" + +// Tests for GraphProfileCalculator. +using testing::ElementsAre; + +namespace mediapipe { +namespace { +using mediapipe::Clock; + +// A Calculator with a fixed Process call latency. +class SleepCalculator : public CalculatorBase { + public: + static absl::Status GetContract(CalculatorContract* cc) { + cc->InputSidePackets().Tag("CLOCK").Set>(); + cc->Inputs().Index(0).SetAny(); + cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0)); + cc->SetTimestampOffset(TimestampDiff(0)); + return absl::OkStatus(); + } + absl::Status Open(CalculatorContext* cc) final { + clock_ = cc->InputSidePackets().Tag("CLOCK").Get>(); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + clock_->Sleep(absl::Milliseconds(5)); + cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value()); + return absl::OkStatus(); + } + std::shared_ptr<::mediapipe::Clock> clock_ = nullptr; +}; +REGISTER_CALCULATOR(SleepCalculator); + +// Tests showing GraphProfileCalculator reporting GraphProfile output packets. +class GraphProfileCalculatorTest : public ::testing::Test { + protected: + void SetUpProfileGraph() { + ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"( + input_stream: "input_packets_0" + node { + calculator: 'SleepCalculator' + input_side_packet: 'CLOCK:sync_clock' + input_stream: 'input_packets_0' + output_stream: 'output_packets_1' + } + node { + calculator: "GraphProfileCalculator" + options: { + [mediapipe.GraphProfileCalculatorOptions.ext]: { + profile_interval: 25000 + } + } + input_stream: "FRAME:output_packets_1" + output_stream: "PROFILE:output_packets_0" + } + )", + &graph_config_)); + } + + static Packet PacketAt(int64 ts) { + return Adopt(new int64(999)).At(Timestamp(ts)); + } + static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); } + static bool IsNone(const Packet& packet) { + return packet.Timestamp() == Timestamp::OneOverPostStream(); + } + // Return the values of the timestamps of a vector of Packets. + static std::vector TimestampValues( + const std::vector& packets) { + std::vector result; + for (const Packet& p : packets) { + result.push_back(p.Timestamp().Value()); + } + return result; + } + + // Runs a CalculatorGraph with a series of packet sets. + // Returns a vector of packets from each graph output stream. + void RunGraph(const std::vector>& input_sets, + std::vector* output_packets) { + // Register output packet observers. + tool::AddVectorSink("output_packets_0", &graph_config_, output_packets); + + // Start running the graph. + std::shared_ptr executor( + new SimulationClockExecutor(3 /*num_threads*/)); + CalculatorGraph graph; + MP_ASSERT_OK(graph.SetExecutor("", executor)); + graph.profiler()->SetClock(executor->GetClock()); + MP_ASSERT_OK(graph.Initialize(graph_config_)); + executor->GetClock()->ThreadStart(); + MP_ASSERT_OK(graph.StartRun({ + {"sync_clock", + Adopt(new std::shared_ptr<::mediapipe::Clock>(executor->GetClock()))}, + })); + + // Send each packet to the graph in the specified order. + for (int t = 0; t < input_sets.size(); t++) { + const std::vector& input_set = input_sets[t]; + for (int i = 0; i < input_set.size(); i++) { + const Packet& packet = input_set[i]; + if (!IsNone(packet)) { + MP_EXPECT_OK(graph.AddPacketToInputStream( + absl::StrCat("input_packets_", i), packet)); + } + executor->GetClock()->Sleep(absl::Milliseconds(10)); + } + } + MP_ASSERT_OK(graph.CloseAllInputStreams()); + executor->GetClock()->Sleep(absl::Milliseconds(100)); + executor->GetClock()->ThreadFinish(); + MP_ASSERT_OK(graph.WaitUntilDone()); + } + + CalculatorGraphConfig graph_config_; +}; + +TEST_F(GraphProfileCalculatorTest, GraphProfile) { + SetUpProfileGraph(); + auto profiler_config = graph_config_.mutable_profiler_config(); + profiler_config->set_enable_profiler(true); + profiler_config->set_trace_enabled(false); + profiler_config->set_trace_log_disabled(true); + profiler_config->set_enable_stream_latency(true); + profiler_config->set_calculator_filter(".*Calculator"); + + // Run the graph with a series of packet sets. + std::vector> input_sets = { + {PacketAt(10000)}, // + {PacketAt(20000)}, // + {PacketAt(30000)}, // + {PacketAt(40000)}, + }; + std::vector output_packets; + RunGraph(input_sets, &output_packets); + + // Validate the output packets. + EXPECT_THAT(TimestampValues(output_packets), // + ElementsAre(10000, 40000)); + + GraphProfile expected_profile = + mediapipe::ParseTextProtoOrDie(R"pb( + calculator_profiles { + name: "GraphProfileCalculator" + open_runtime: 0 + process_runtime { total: 0 count: 3 } + process_input_latency { total: 15000 count: 3 } + process_output_latency { total: 15000 count: 3 } + input_stream_profiles { + name: "output_packets_1" + back_edge: false + latency { total: 0 count: 3 } + } + } + calculator_profiles { + name: "SleepCalculator" + open_runtime: 0 + process_runtime { total: 15000 count: 3 } + process_input_latency { total: 0 count: 3 } + process_output_latency { total: 15000 count: 3 } + input_stream_profiles { + name: "input_packets_0" + back_edge: false + latency { total: 0 count: 3 } + } + })pb"); + + EXPECT_THAT(output_packets[1].Get(), + mediapipe::EqualsProto(expected_profile)); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/calculators/image/bilateral_filter_calculator.cc b/mediapipe/calculators/image/bilateral_filter_calculator.cc index 3d878bffc..6bb43dc00 100644 --- a/mediapipe/calculators/image/bilateral_filter_calculator.cc +++ b/mediapipe/calculators/image/bilateral_filter_calculator.cc @@ -240,7 +240,7 @@ absl::Status BilateralFilterCalculator::RenderCpu(CalculatorContext* cc) { auto input_mat = mediapipe::formats::MatView(&input_frame); // Only 1 or 3 channel images supported by OpenCV. - if ((input_mat.channels() == 1 || input_mat.channels() == 3)) { + if (!(input_mat.channels() == 1 || input_mat.channels() == 3)) { return absl::InternalError( "CPU filtering supports only 1 or 3 channel input images."); } diff --git a/mediapipe/calculators/image/image_clone_calculator.cc b/mediapipe/calculators/image/image_clone_calculator.cc index 107c42b92..1e76848b1 100644 --- a/mediapipe/calculators/image/image_clone_calculator.cc +++ b/mediapipe/calculators/image/image_clone_calculator.cc @@ -36,7 +36,7 @@ using GpuBuffer = mediapipe::GpuBuffer; // stored on the target storage (CPU vs GPU) specified in the calculator option. // // The clone shares ownership of the input pixel data on the existing storage. -// If the target storage is diffrent from the existing one, then the data is +// If the target storage is different from the existing one, then the data is // further copied there. // // Example usage: diff --git a/mediapipe/calculators/tensor/inference_calculator.cc b/mediapipe/calculators/tensor/inference_calculator.cc index 11256a338..46e0f928c 100644 --- a/mediapipe/calculators/tensor/inference_calculator.cc +++ b/mediapipe/calculators/tensor/inference_calculator.cc @@ -33,7 +33,7 @@ class InferenceCalculatorSelectorImpl absl::StatusOr GetConfig( const CalculatorGraphConfig::Node& subgraph_node) { const auto& options = - Subgraph::GetOptions<::mediapipe::InferenceCalculatorOptions>( + Subgraph::GetOptions( subgraph_node); std::vector impls; const bool should_use_gpu = diff --git a/mediapipe/calculators/tensor/inference_calculator.h b/mediapipe/calculators/tensor/inference_calculator.h index 9fe06181c..d354790ad 100644 --- a/mediapipe/calculators/tensor/inference_calculator.h +++ b/mediapipe/calculators/tensor/inference_calculator.h @@ -99,8 +99,13 @@ class InferenceCalculator : public NodeIntf { kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"}; static constexpr SideInput::Optional kSideInModel{"MODEL"}; static constexpr Output> kOutTensors{"TENSORS"}; + static constexpr SideInput::Optional kNnApiDelegateCacheDir{ + "NNAPI_CACHE_DIR"}; + static constexpr SideInput::Optional kNnApiDelegateModelToken{ + "NNAPI_MODEL_TOKEN"}; MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel, - kOutTensors); + kOutTensors, kNnApiDelegateCacheDir, + kNnApiDelegateModelToken); protected: using TfLiteDelegatePtr = diff --git a/mediapipe/calculators/tensor/inference_calculator.proto b/mediapipe/calculators/tensor/inference_calculator.proto index e0b538a91..6718901a5 100644 --- a/mediapipe/calculators/tensor/inference_calculator.proto +++ b/mediapipe/calculators/tensor/inference_calculator.proto @@ -67,9 +67,32 @@ message InferenceCalculatorOptions { // Only available for OpenCL delegate on Android. // Kernel caching will only be enabled if this path is set. optional string cached_kernel_path = 2; + + // Encapsulated compilation/runtime tradeoffs. + enum InferenceUsage { + UNSPECIFIED = 0; + + // InferenceRunner will be used only once. Therefore, it is important to + // minimize bootstrap time as well. + FAST_SINGLE_ANSWER = 1; + + // Prefer maximizing the throughput. Same inference runner will be used + // repeatedly on different inputs. + SUSTAINED_SPEED = 2; + } + optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED]; } + // Android only. - message Nnapi {} + message Nnapi { + // Directory to store compilation cache. If unspecified, NNAPI will not + // try caching the compilation. + optional string cache_dir = 1; + // Unique token identifying the model. It is the caller's responsibility + // to ensure there is no clash of the tokens. If unspecified, NNAPI will + // not try caching the compilation. + optional string model_token = 2; + } message Xnnpack { // Number of threads for XNNPACK delegate. (By default, calculator tries // to choose optimal number of threads depending on the device.) diff --git a/mediapipe/calculators/tensor/inference_calculator_cpu.cc b/mediapipe/calculators/tensor/inference_calculator_cpu.cc index 0299ab526..4995d9d4e 100644 --- a/mediapipe/calculators/tensor/inference_calculator_cpu.cc +++ b/mediapipe/calculators/tensor/inference_calculator_cpu.cc @@ -181,9 +181,21 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) { // Attempt to use NNAPI. // If not supported, the default CPU delegate will be created and used. interpreter_->SetAllowFp16PrecisionForFp32(1); - delegate_ = TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) { - // No need to free according to tflite::NnApiDelegate() documentation. - }); + tflite::StatefulNnApiDelegate::Options options; + const auto& nnapi = calculator_opts.delegate().nnapi(); + // Set up cache_dir and model_token for NNAPI compilation cache. + options.cache_dir = + nnapi.has_cache_dir() ? nnapi.cache_dir().c_str() : nullptr; + if (!kNnApiDelegateCacheDir(cc).IsEmpty()) { + options.cache_dir = kNnApiDelegateCacheDir(cc).Get().c_str(); + } + options.model_token = + nnapi.has_model_token() ? nnapi.model_token().c_str() : nullptr; + if (!kNnApiDelegateModelToken(cc).IsEmpty()) { + options.model_token = kNnApiDelegateModelToken(cc).Get().c_str(); + } + delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options), + [](TfLiteDelegate*) {}); RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), kTfLiteOk); return absl::OkStatus(); diff --git a/mediapipe/calculators/tensor/inference_calculator_gl.cc b/mediapipe/calculators/tensor/inference_calculator_gl.cc index 5769df20e..4072e1d87 100644 --- a/mediapipe/calculators/tensor/inference_calculator_gl.cc +++ b/mediapipe/calculators/tensor/inference_calculator_gl.cc @@ -18,6 +18,7 @@ #include #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "mediapipe/calculators/tensor/inference_calculator.h" #include "mediapipe/util/tflite/config.h" @@ -65,6 +66,8 @@ class InferenceCalculatorGlImpl bool allow_precision_loss_ = false; mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api tflite_gpu_runner_api_; + mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage + tflite_gpu_runner_usage_; #endif // MEDIAPIPE_TFLITE_GL_INFERENCE #if MEDIAPIPE_TFLITE_GPU_SUPPORTED @@ -96,6 +99,7 @@ absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) { options.delegate().gpu().use_advanced_gpu_api(); allow_precision_loss_ = options.delegate().gpu().allow_precision_loss(); tflite_gpu_runner_api_ = options.delegate().gpu().api(); + tflite_gpu_runner_usage_ = options.delegate().gpu().usage(); use_kernel_caching_ = use_advanced_gpu_api_ && options.delegate().gpu().has_cached_kernel_path(); use_gpu_delegate_ = !use_advanced_gpu_api_; @@ -253,9 +257,27 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner( : tflite::gpu::InferencePriority::MAX_PRECISION; options.priority2 = tflite::gpu::InferencePriority::AUTO; options.priority3 = tflite::gpu::InferencePriority::AUTO; - options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; + switch (tflite_gpu_runner_usage_) { + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu:: + FAST_SINGLE_ANSWER: { + options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER; + break; + } + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu:: + SUSTAINED_SPEED: { + options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; + break; + } + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::UNSPECIFIED: { + return absl::InternalError("inference usage need to be specified."); + } + } tflite_gpu_runner_ = std::make_unique(options); switch (tflite_gpu_runner_api_) { + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: { + // Do not need to force any specific API. + break; + } case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::OPENGL: { tflite_gpu_runner_->ForceOpenGL(); break; @@ -264,10 +286,6 @@ absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner( tflite_gpu_runner_->ForceOpenCL(); break; } - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: { - // Do not need to force any specific API. - break; - } } MP_RETURN_IF_ERROR(tflite_gpu_runner_->InitializeWithModel( model, op_resolver, /*allow_quant_ops=*/true)); diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 0dbbd57da..9d00b069a 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -864,6 +864,7 @@ cc_test( "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", + "//mediapipe/framework:timestamp", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index ddb042e6a..5d8008a29 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -243,11 +243,6 @@ class PackMediaSequenceCalculator : public CalculatorBase { } } - if (cc->Outputs().HasTag(kSequenceExampleTag)) { - cc->Outputs() - .Tag(kSequenceExampleTag) - .SetNextTimestampBound(Timestamp::Max()); - } return absl::OkStatus(); } @@ -305,7 +300,9 @@ class PackMediaSequenceCalculator : public CalculatorBase { if (cc->Outputs().HasTag(kSequenceExampleTag)) { cc->Outputs() .Tag(kSequenceExampleTag) - .Add(sequence_.release(), Timestamp::PostStream()); + .Add(sequence_.release(), options.output_as_zero_timestamp() + ? Timestamp(0ll) + : Timestamp::PostStream()); } sequence_.reset(); diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto index 695eb6b5e..6ba09fb16 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto @@ -65,4 +65,7 @@ message PackMediaSequenceCalculatorOptions { // If true, will return an error status if an output sequence would be too // many bytes to serialize. optional bool skip_large_sequences = 7 [default = true]; + + // If true/false, outputs the SequenceExample at timestamp 0/PostStream. + optional bool output_as_zero_timestamp = 8 [default = false]; } diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc index c163cebcd..78bc1f24e 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc @@ -29,6 +29,7 @@ #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/timestamp.h" #include "mediapipe/util/sequence/media_sequence.h" #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/feature.pb.h" @@ -43,8 +44,9 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test { protected: void SetUpCalculator(const std::vector& input_streams, const tf::Features& features, - bool output_only_if_all_present, - bool replace_instead_of_append) { + const bool output_only_if_all_present, + const bool replace_instead_of_append, + const bool output_as_zero_timestamp = false) { CalculatorGraphConfig::Node config; config.set_calculator("PackMediaSequenceCalculator"); config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence"); @@ -57,6 +59,7 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test { *options->mutable_context_feature_map() = features; options->set_output_only_if_all_present(output_only_if_all_present); options->set_replace_data_instead_of_append(replace_instead_of_append); + options->set_output_as_zero_timestamp(output_as_zero_timestamp); runner_ = ::absl::make_unique(config); } @@ -194,6 +197,29 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) { } } +TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) { + SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true); + auto input_sequence = ::absl::make_unique(); + + int num_timesteps = 2; + for (int i = 0; i < num_timesteps; ++i) { + auto vf_ptr = ::absl::make_unique>(2, 2 << i); + runner_->MutableInputs() + ->Tag("FLOAT_FEATURE_TEST") + .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); + } + + runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; + ASSERT_EQ(1, output_packets.size()); + EXPECT_EQ(output_packets[0].Timestamp().Value(), 0ll); +} + TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) { SetUpCalculator( {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"}, diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index 9ec556987..633300a73 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -292,6 +292,8 @@ class TfLiteInferenceCalculator : public CalculatorBase { bool allow_precision_loss_ = false; mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api tflite_gpu_runner_api_; + mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::InferenceUsage + tflite_gpu_runner_usage_; bool use_kernel_caching_ = false; std::string cached_kernel_filename_; @@ -377,6 +379,7 @@ absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) { options.delegate().gpu().use_advanced_gpu_api(); allow_precision_loss_ = options.delegate().gpu().allow_precision_loss(); tflite_gpu_runner_api_ = options.delegate().gpu().api(); + tflite_gpu_runner_usage_ = options.delegate().gpu().usage(); use_kernel_caching_ = use_advanced_gpu_api_ && options.delegate().gpu().has_cached_kernel_path(); @@ -733,7 +736,23 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner( : tflite::gpu::InferencePriority::MAX_PRECISION; options.priority2 = tflite::gpu::InferencePriority::AUTO; options.priority3 = tflite::gpu::InferencePriority::AUTO; - options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; + switch (tflite_gpu_runner_usage_) { + case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu:: + FAST_SINGLE_ANSWER: { + options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER; + break; + } + case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu:: + SUSTAINED_SPEED: { + options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; + break; + } + case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu:: + UNSPECIFIED: { + return absl::InternalError("inference usage need to be specified."); + } + } + tflite_gpu_runner_ = std::make_unique(options); switch (tflite_gpu_runner_api_) { case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::OPENGL: { @@ -878,11 +897,15 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) { // Attempt to use NNAPI. // If not supported, the default CPU delegate will be created and used. interpreter_->SetAllowFp16PrecisionForFp32(1); - delegate_ = - TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) { - // No need to free according to tflite::NnApiDelegate() - // documentation. - }); + tflite::StatefulNnApiDelegate::Options options; + const auto& nnapi = calculator_opts.delegate().nnapi(); + // Set up cache_dir and model_token for NNAPI compilation cache. + if (nnapi.has_cache_dir() && nnapi.has_model_token()) { + options.cache_dir = nnapi.cache_dir().c_str(); + options.model_token = nnapi.model_token().c_str(); + } + delegate_ = TfLiteDelegatePtr(new tflite::StatefulNnApiDelegate(options), + [](TfLiteDelegate*) {}); RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), kTfLiteOk); return absl::OkStatus(); diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.proto b/mediapipe/calculators/tflite/tflite_inference_calculator.proto index 02dc20831..3b4d2896e 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.proto +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.proto @@ -67,9 +67,31 @@ message TfLiteInferenceCalculatorOptions { // Only available for OpenCL delegate on Android. // Kernel caching will only be enabled if this path is set. optional string cached_kernel_path = 2; + + // Encapsulated compilation/runtime tradeoffs. + enum InferenceUsage { + UNSPECIFIED = 0; + + // InferenceRunner will be used only once. Therefore, it is important to + // minimize bootstrap time as well. + FAST_SINGLE_ANSWER = 1; + + // Prefer maximizing the throughput. Same inference runner will be used + // repeatedly on different inputs. + SUSTAINED_SPEED = 2; + } + optional InferenceUsage usage = 5 [default = SUSTAINED_SPEED]; } // Android only. - message Nnapi {} + message Nnapi { + // Directory to store compilation cache. If unspecified, NNAPI will not + // try caching the compilation. + optional string cache_dir = 1; + // Unique token identifying the model. It is the caller's responsibility + // to ensure there is no clash of the tokens. If unspecified, NNAPI will + // not try caching the compilation. + optional string model_token = 2; + } message Xnnpack { // Number of threads for XNNPACK delegate. (By default, calculator tries // to choose optimal number of threads depending on the device.) diff --git a/mediapipe/examples/android/solutions/BUILD b/mediapipe/examples/android/solutions/BUILD new file mode 100644 index 000000000..1ba23afe6 --- /dev/null +++ b/mediapipe/examples/android/solutions/BUILD @@ -0,0 +1,21 @@ +# Copyright 2021 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +filegroup( + name = "resource_files", + srcs = glob(["res/**"]), + visibility = ["//mediapipe/examples/android/solutions:__subpackages__"], +) diff --git a/mediapipe/examples/android/solutions/build.gradle b/mediapipe/examples/android/solutions/build.gradle new file mode 100644 index 000000000..691e41013 --- /dev/null +++ b/mediapipe/examples/android/solutions/build.gradle @@ -0,0 +1,24 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + classpath "com.android.tools.build:gradle:4.2.0" + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + mavenCentral() + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/mediapipe/examples/android/solutions/gradle.properties b/mediapipe/examples/android/solutions/gradle.properties new file mode 100644 index 000000000..c09e1e3b0 --- /dev/null +++ b/mediapipe/examples/android/solutions/gradle.properties @@ -0,0 +1,17 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app"s APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e708b1c02 Binary files /dev/null and b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar differ diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..442d9132e --- /dev/null +++ b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/mediapipe/examples/android/solutions/gradlew b/mediapipe/examples/android/solutions/gradlew new file mode 100755 index 000000000..4f906e0c8 --- /dev/null +++ b/mediapipe/examples/android/solutions/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/mediapipe/examples/android/solutions/gradlew.bat b/mediapipe/examples/android/solutions/gradlew.bat new file mode 100755 index 000000000..ac1b06f93 --- /dev/null +++ b/mediapipe/examples/android/solutions/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/mediapipe/examples/android/solutions/hands/build.gradle b/mediapipe/examples/android/solutions/hands/build.gradle new file mode 100644 index 000000000..27629fd5d --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/build.gradle @@ -0,0 +1,50 @@ +plugins { + id 'com.android.application' +} + +android { + compileSdkVersion 30 + buildToolsVersion "30.0.3" + + defaultConfig { + applicationId "com.google.mediapipe.apps.hands" + minSdkVersion 21 + targetSdkVersion 30 + versionCode 1 + versionName "1.0" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar', '*.aar']) + implementation 'androidx.appcompat:appcompat:1.3.0' + implementation 'com.google.android.material:material:1.3.0' + implementation 'androidx.constraintlayout:constraintlayout:2.0.4' + testImplementation 'junit:junit:4.+' + androidTestImplementation 'androidx.test.ext:junit:1.1.2' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' + // MediaPipe hands solution API and solution-core. + implementation 'com.google.mediapipe:solution-core:latest.release' + implementation 'com.google.mediapipe:hands:latest.release' + // MediaPipe deps + implementation 'com.google.flogger:flogger:latest.release' + implementation 'com.google.flogger:flogger-system-backend:latest.release' + implementation 'com.google.guava:guava:27.0.1-android' + implementation 'com.google.protobuf:protobuf-java:3.11.4' + // CameraX core library + def camerax_version = "1.0.0-beta10" + implementation "androidx.camera:camera-core:$camerax_version" + implementation "androidx.camera:camera-camera2:$camerax_version" + implementation "androidx.camera:camera-lifecycle:$camerax_version" +} diff --git a/mediapipe/examples/android/solutions/hands/proguard-rules.pro b/mediapipe/examples/android/solutions/hands/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/mediapipe/examples/android/solutions/hands/src/main/AndroidManifest.xml b/mediapipe/examples/android/solutions/hands/src/main/AndroidManifest.xml new file mode 100644 index 000000000..0d70af7ff --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/AndroidManifest.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/mediapipe/examples/android/solutions/hands/src/main/BUILD b/mediapipe/examples/android/solutions/hands/src/main/BUILD new file mode 100644 index 000000000..c4bb724e0 --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/BUILD @@ -0,0 +1,40 @@ +# Copyright 2021 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:private"]) + +android_binary( + name = "hands", + srcs = glob(["**/*.java"]), + custom_package = "com.google.mediapipe.examples.hands", + manifest = "AndroidManifest.xml", + manifest_values = { + "applicationId": "com.google.mediapipe.examples.hands", + }, + multidex = "native", + resource_files = ["//mediapipe/examples/android/solutions:resource_files"], + deps = [ + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/solutioncore:camera_input", + "//mediapipe/java/com/google/mediapipe/solutioncore:mediapipe_jni_lib", + "//mediapipe/java/com/google/mediapipe/solutioncore:solution_rendering", + "//mediapipe/java/com/google/mediapipe/solutions/hands", + "//third_party:androidx_appcompat", + "//third_party:androidx_constraint_layout", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:com_google_guava_guava", + ], +) diff --git a/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java new file mode 100644 index 000000000..ec61110ed --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultGlRenderer.java @@ -0,0 +1,129 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.examples.hands; + +import android.opengl.GLES20; +import android.opengl.Matrix; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.solutioncore.ResultGlBoundary; +import com.google.mediapipe.solutioncore.ResultGlRenderer; +import com.google.mediapipe.solutions.hands.Hands; +import com.google.mediapipe.solutions.hands.HandsResult; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.List; + +/** A custom implementation of {@link ResultGlRenderer} to render MediaPope Hands results. */ +public class HandsResultGlRenderer implements ResultGlRenderer { + private static final String TAG = "HandsResultGlRenderer"; + + private static final float CONNECTION_THICKNESS = 20.0f; + private static final String VERTEX_SHADER = + "uniform mat4 uTransformMatrix;\n" + + "attribute vec4 vPosition;\n" + + "void main() {\n" + + " gl_Position = uTransformMatrix * vPosition;\n" + + "}"; + private static final String FRAGMENT_SHADER = + "precision mediump float;\n" + + "void main() {\n" + + " gl_FragColor = vec4(0, 1, 0, 1);\n" + + "}"; + private int program; + private int positionHandle; + private int transformMatrixHandle; + private final float[] transformMatrix = new float[16]; + private FloatBuffer vertexBuffer; + + private int loadShader(int type, String shaderCode) { + int shader = GLES20.glCreateShader(type); + GLES20.glShaderSource(shader, shaderCode); + GLES20.glCompileShader(shader); + return shader; + } + + @Override + public void setupRendering() { + program = GLES20.glCreateProgram(); + int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, VERTEX_SHADER); + int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT_SHADER); + GLES20.glAttachShader(program, vertexShader); + GLES20.glAttachShader(program, fragmentShader); + GLES20.glLinkProgram(program); + positionHandle = GLES20.glGetAttribLocation(program, "vPosition"); + transformMatrixHandle = GLES20.glGetUniformLocation(program, "uTransformMatrix"); + } + + @Override + public void renderResult(HandsResult result, ResultGlBoundary boundary) { + if (result == null) { + return; + } + GLES20.glUseProgram(program); + // Sets the transform matrix to align the result rendering with the scaled output texture. + Matrix.setIdentityM(transformMatrix, 0); + Matrix.scaleM( + transformMatrix, + 0, + 2 / (boundary.right() - boundary.left()), + 2 / (boundary.top() - boundary.bottom()), + 1.0f); + GLES20.glUniformMatrix4fv(transformMatrixHandle, 1, false, transformMatrix, 0); + GLES20.glLineWidth(CONNECTION_THICKNESS); + + int numHands = result.multiHandLandmarks().size(); + for (int i = 0; i < numHands; ++i) { + drawLandmarks(result.multiHandLandmarks().get(i).getLandmarkList()); + } + } + + /** + * Calls this to delete the shader program. + * + *

This is only necessary if one wants to release the program while keeping the context around. + */ + public void release() { + GLES20.glDeleteProgram(program); + } + + // TODO: Better hand landmark and hand connection drawing. + private void drawLandmarks(List handLandmarkList) { + for (Hands.Connection c : Hands.HAND_CONNECTIONS) { + float[] vertex = new float[4]; + NormalizedLandmark start = handLandmarkList.get(c.start()); + vertex[0] = normalizedLandmarkValue(start.getX()); + vertex[1] = normalizedLandmarkValue(start.getY()); + NormalizedLandmark end = handLandmarkList.get(c.end()); + vertex[2] = normalizedLandmarkValue(end.getX()); + vertex[3] = normalizedLandmarkValue(end.getY()); + vertexBuffer = + ByteBuffer.allocateDirect(vertex.length * 4) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + .put(vertex); + vertexBuffer.position(0); + GLES20.glEnableVertexAttribArray(positionHandle); + GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer); + GLES20.glDrawArrays(GLES20.GL_LINES, 0, 2); + } + } + + // Normalizes the value from the landmark value range:[0, 1] to the standard OpenGL coordinate + // value range: [-1, 1]. + private float normalizedLandmarkValue(float value) { + return value * 2 - 1; + } +} diff --git a/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java new file mode 100644 index 000000000..35dbc1848 --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/HandsResultImageView.java @@ -0,0 +1,95 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.examples.hands; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.widget.ImageView; +import com.google.mediapipe.formats.proto.LandmarkProto; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.solutions.hands.Hands; +import com.google.mediapipe.solutions.hands.HandsResult; +import java.util.List; + +/** An ImageView implementation for displaying MediaPipe Hands results. */ +public class HandsResultImageView extends ImageView { + private static final String TAG = "HandsResultImageView"; + + private static final int LANDMARK_COLOR = Color.RED; + private static final int LANDMARK_RADIUS = 15; + private static final int CONNECTION_COLOR = Color.GREEN; + private static final int CONNECTION_THICKNESS = 10; + + public HandsResultImageView(Context context) { + super(context); + setScaleType(ImageView.ScaleType.FIT_CENTER); + } + + /** + * Sets a {@link HandsResult} to render. + * + * @param result a {@link HandsResult} object that contains the solution outputs and the input + * {@link Bitmap}. + */ + public void setHandsResult(HandsResult result) { + if (result == null) { + return; + } + Bitmap bmInput = result.inputBitmap(); + int width = bmInput.getWidth(); + int height = bmInput.getHeight(); + Bitmap bmOutput = Bitmap.createBitmap(width, height, bmInput.getConfig()); + Canvas canvas = new Canvas(bmOutput); + + canvas.drawBitmap(bmInput, new Matrix(), null); + int numHands = result.multiHandLandmarks().size(); + for (int i = 0; i < numHands; ++i) { + drawLandmarksOnCanvas( + result.multiHandLandmarks().get(i).getLandmarkList(), canvas, width, height); + } + postInvalidate(); + setImageBitmap(bmOutput); + } + + // TODO: Better hand landmark and hand connection drawing. + private void drawLandmarksOnCanvas( + List handLandmarkList, Canvas canvas, int width, int height) { + // Draw connections. + for (Hands.Connection c : Hands.HAND_CONNECTIONS) { + Paint connectionPaint = new Paint(); + connectionPaint.setColor(CONNECTION_COLOR); + connectionPaint.setStrokeWidth(CONNECTION_THICKNESS); + NormalizedLandmark start = handLandmarkList.get(c.start()); + NormalizedLandmark end = handLandmarkList.get(c.end()); + canvas.drawLine( + start.getX() * width, + start.getY() * height, + end.getX() * width, + end.getY() * height, + connectionPaint); + } + Paint landmarkPaint = new Paint(); + landmarkPaint.setColor(LANDMARK_COLOR); + // Draw landmarks. + for (LandmarkProto.NormalizedLandmark landmark : handLandmarkList) { + canvas.drawCircle( + landmark.getX() * width, landmark.getY() * height, LANDMARK_RADIUS, landmarkPaint); + } + } +} diff --git a/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java new file mode 100644 index 000000000..8828c0240 --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/java/com/google/mediapipe/examples/hands/MainActivity.java @@ -0,0 +1,241 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.examples.hands; + +import android.content.Intent; +import android.graphics.Bitmap; +import android.os.Bundle; +import android.provider.MediaStore; +import androidx.appcompat.app.AppCompatActivity; +import android.util.Log; +import android.view.View; +import android.widget.Button; +import android.widget.FrameLayout; +import androidx.activity.result.ActivityResultLauncher; +import androidx.activity.result.contract.ActivityResultContracts; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.solutioncore.CameraInput; +import com.google.mediapipe.solutioncore.SolutionGlSurfaceView; +import com.google.mediapipe.solutions.hands.HandLandmark; +import com.google.mediapipe.solutions.hands.Hands; +import com.google.mediapipe.solutions.hands.HandsOptions; +import com.google.mediapipe.solutions.hands.HandsResult; +import java.io.IOException; + +/** Main activity of MediaPipe Hands app. */ +public class MainActivity extends AppCompatActivity { + private static final String TAG = "MainActivity"; + + private Hands hands; + private int mode = HandsOptions.STATIC_IMAGE_MODE; + // Image demo UI and image loader components. + private Button loadImageButton; + private ActivityResultLauncher imageGetter; + private HandsResultImageView imageView; + + // Live camera demo UI and camera components. + private Button startCameraButton; + private CameraInput cameraInput; + private SolutionGlSurfaceView glSurfaceView; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + setupStaticImageDemoUiComponents(); + setupLiveDemoUiComponents(); + } + + @Override + protected void onResume() { + super.onResume(); + if (mode == HandsOptions.STREAMING_MODE) { + // Restarts the camera and the opengl surface rendering. + cameraInput = new CameraInput(this); + cameraInput.setCameraNewFrameListener(textureFrame -> hands.send(textureFrame)); + glSurfaceView.post(this::startCamera); + glSurfaceView.setVisibility(View.VISIBLE); + } + } + + @Override + protected void onPause() { + super.onPause(); + if (mode == HandsOptions.STREAMING_MODE) { + stopLiveDemo(); + } + } + + /** Sets up the UI components for the static image demo. */ + private void setupStaticImageDemoUiComponents() { + // The Intent to access gallery and read images as bitmap. + imageGetter = + registerForActivityResult( + new ActivityResultContracts.StartActivityForResult(), + result -> { + Intent resultIntent = result.getData(); + if (resultIntent != null) { + if (result.getResultCode() == RESULT_OK) { + Bitmap bitmap = null; + try { + bitmap = + MediaStore.Images.Media.getBitmap( + this.getContentResolver(), resultIntent.getData()); + } catch (IOException e) { + Log.e(TAG, "Bitmap reading error:" + e); + } + if (bitmap != null) { + hands.send(bitmap); + } + } + } + }); + loadImageButton = (Button) findViewById(R.id.button_load_picture); + loadImageButton.setOnClickListener( + new View.OnClickListener() { + @Override + public void onClick(View v) { + if (mode == HandsOptions.STREAMING_MODE) { + stopLiveDemo(); + } + if (hands == null || mode != HandsOptions.STATIC_IMAGE_MODE) { + setupStaticImageModePipeline(); + } + // Reads images from gallery. + Intent gallery = + new Intent(Intent.ACTION_PICK, MediaStore.Images.Media.INTERNAL_CONTENT_URI); + imageGetter.launch(gallery); + } + }); + imageView = new HandsResultImageView(this); + } + + /** The core MediaPipe Hands setup workflow for its static image mode. */ + private void setupStaticImageModePipeline() { + // Initializes a new MediaPipe Hands instance in the static image mode. + mode = HandsOptions.STATIC_IMAGE_MODE; + if (hands != null) { + hands.close(); + } + hands = new Hands(this, HandsOptions.builder().setMode(mode).build()); + + // Connects MediaPipe Hands to the user-defined HandsResultImageView. + hands.setResultListener( + handsResult -> { + logWristLandmark(handsResult, /*showPixelValues=*/ true); + runOnUiThread(() -> imageView.setHandsResult(handsResult)); + }); + hands.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe hands error:" + message)); + + // Updates the preview layout. + FrameLayout frameLayout = (FrameLayout) findViewById(R.id.preview_display_layout); + frameLayout.removeAllViewsInLayout(); + imageView.setImageDrawable(null); + frameLayout.addView(imageView); + imageView.setVisibility(View.VISIBLE); + } + + /** Sets up the UI components for the live demo with camera input. */ + private void setupLiveDemoUiComponents() { + startCameraButton = (Button) findViewById(R.id.button_start_camera); + startCameraButton.setOnClickListener( + new View.OnClickListener() { + @Override + public void onClick(View v) { + if (hands == null || mode != HandsOptions.STREAMING_MODE) { + setupStreamingModePipeline(); + } + } + }); + } + + /** The core MediaPipe Hands setup workflow for its streaming mode. */ + private void setupStreamingModePipeline() { + // Initializes a new MediaPipe Hands instance in the streaming mode. + mode = HandsOptions.STREAMING_MODE; + if (hands != null) { + hands.close(); + } + hands = new Hands(this, HandsOptions.builder().setMode(mode).build()); + hands.setErrorListener((message, e) -> Log.e(TAG, "MediaPipe hands error:" + message)); + + // Initializes a new CameraInput instance and connects it to MediaPipe Hands. + cameraInput = new CameraInput(this); + cameraInput.setCameraNewFrameListener(textureFrame -> hands.send(textureFrame)); + + // Initalizes a new Gl surface view with a user-defined HandsResultGlRenderer. + glSurfaceView = + new SolutionGlSurfaceView<>(this, hands.getGlContext(), hands.getGlMajorVersion()); + glSurfaceView.setSolutionResultRenderer(new HandsResultGlRenderer()); + glSurfaceView.setRenderInputImage(true); + hands.setResultListener( + handsResult -> { + logWristLandmark(handsResult, /*showPixelValues=*/ false); + glSurfaceView.setRenderData(handsResult); + glSurfaceView.requestRender(); + }); + + // The runnable to start camera after the gl surface view is attached. + glSurfaceView.post(this::startCamera); + + // Updates the preview layout. + FrameLayout frameLayout = (FrameLayout) findViewById(R.id.preview_display_layout); + imageView.setVisibility(View.GONE); + frameLayout.removeAllViewsInLayout(); + frameLayout.addView(glSurfaceView); + glSurfaceView.setVisibility(View.VISIBLE); + frameLayout.requestLayout(); + } + + private void startCamera() { + cameraInput.start( + this, + hands.getGlContext(), + CameraInput.CameraFacing.FRONT, + glSurfaceView.getWidth(), + glSurfaceView.getHeight()); + } + + private void stopLiveDemo() { + if (cameraInput != null) { + cameraInput.stop(); + } + if (glSurfaceView != null) { + glSurfaceView.setVisibility(View.GONE); + } + } + + private void logWristLandmark(HandsResult result, boolean showPixelValues) { + NormalizedLandmark wristLandmark = Hands.getHandLandmark(result, 0, HandLandmark.WRIST); + // For Bitmaps, show the pixel values. For texture inputs, show the normoralized cooridanates. + if (showPixelValues) { + int width = result.inputBitmap().getWidth(); + int height = result.inputBitmap().getHeight(); + Log.i( + TAG, + "MediaPipe Hand wrist coordinates (pixel values): x= " + + wristLandmark.getX() * width + + " y=" + + wristLandmark.getY() * height); + } else { + Log.i( + TAG, + "MediaPipe Hand wrist normalized coordinates (value range: [0, 1]): x= " + + wristLandmark.getX() + + " y=" + + wristLandmark.getY()); + } + } +} diff --git a/mediapipe/examples/android/solutions/hands/src/main/res b/mediapipe/examples/android/solutions/hands/src/main/res new file mode 120000 index 000000000..fc8850136 --- /dev/null +++ b/mediapipe/examples/android/solutions/hands/src/main/res @@ -0,0 +1 @@ +../../../res \ No newline at end of file diff --git a/mediapipe/examples/android/solutions/res/drawable-v24/ic_launcher_foreground.xml b/mediapipe/examples/android/solutions/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..c7bd21dbd --- /dev/null +++ b/mediapipe/examples/android/solutions/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/mediapipe/examples/android/solutions/res/drawable/ic_launcher_background.xml b/mediapipe/examples/android/solutions/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..01f0af0ad --- /dev/null +++ b/mediapipe/examples/android/solutions/res/drawable/ic_launcher_background.xml @@ -0,0 +1,74 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mediapipe/examples/android/solutions/res/layout/activity_main.xml b/mediapipe/examples/android/solutions/res/layout/activity_main.xml new file mode 100644 index 000000000..e14f12871 --- /dev/null +++ b/mediapipe/examples/android/solutions/res/layout/activity_main.xml @@ -0,0 +1,35 @@ + + + +