diff --git a/gradio/demo.py b/gradio/demo.py index a158423b1..d0a1967d4 100644 --- a/gradio/demo.py +++ b/gradio/demo.py @@ -1,5 +1,13 @@ import mediapipe as mp import gradio as gr +import cv2 +import torch + + +# Images +torch.hub.download_url_to_file('https://artbreeder.b-cdn.net/imgs/c789e54661bfb432c5522a36553f.jpeg', 'face1.jpg') +torch.hub.download_url_to_file('https://artbreeder.b-cdn.net/imgs/c86622e8cb58d490e35b01cb9996.jpeg', 'face2.jpg') + mp_face_mesh = mp.solutions.face_mesh # Prepare DrawingSpec for drawing the face landmarks later. @@ -16,16 +24,28 @@ def inference(image): # Convert the BGR image to RGB and process it with MediaPipe Face Mesh. results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) - # Draw face landmarks of each face. - print(f'Face landmarks of {name}:') - if not results.multi_face_landmarks: - continue annotated_image = image.copy() for face_landmarks in results.multi_face_landmarks: - mp_drawing.draw_landmarks( - image=annotated_image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACE_CONNECTIONS, - landmark_drawing_spec=drawing_spec, - connection_drawing_spec=drawing_spec) - return annotated_image \ No newline at end of file + mp_drawing.draw_landmarks( + image=annotated_image, + landmark_list=face_landmarks, + connections=mp_face_mesh.FACE_CONNECTIONS, + landmark_drawing_spec=drawing_spec, + connection_drawing_spec=drawing_spec) + return annotated_image + +title = "Face Mesh" +description = "demo for Face Mesh. To use it, simply upload your image, or click one of the examples to load them. Read more at the links below." +article = "

Attention Mesh: High-fidelity Face Mesh Prediction in Real-time | Github Repo

" + +gr.Interface( + inference, + [gr.inputs.Image(label="Input")], + gr.outputs.Image(type="pil", label="Output"), + title=title, + description=description, + article=article, + examples=[ + ["face1.jpg"], + ["face2.jpg"] + ]).launch(debug=True) \ No newline at end of file diff --git a/mediapipe/BUILD b/mediapipe/BUILD deleted file mode 100644 index 1171ea6f0..000000000 --- a/mediapipe/BUILD +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) # Apache 2.0 - -# Note: yes, these need to use "//external:android/crosstool", not -# @androidndk//:default_crosstool. - -config_setting( - name = "android", - values = {"crosstool_top": "//external:android/crosstool"}, - visibility = ["//visibility:public"], -) - -config_setting( - name = "android_x86", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "x86", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "android_x86_64", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "x86_64", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "android_armeabi", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "armeabi", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "android_arm", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "armeabi-v7a", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "android_arm64", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "arm64-v8a", - }, - visibility = ["//visibility:public"], -) - -# Note: this cannot just match "apple_platform_type": "macos" because that option -# defaults to "macos" even when building on Linux! -alias( - name = "macos", - actual = select({ - ":macos_i386": ":macos_i386", - ":macos_x86_64": ":macos_x86_64", - "//conditions:default": ":macos_i386", # Arbitrarily chosen from above. - }), - visibility = ["//visibility:public"], -) - -# Note: this also matches on crosstool_top so that it does not produce ambiguous -# selectors when used together with "android". -config_setting( - name = "ios", - values = { - "crosstool_top": "@bazel_tools//tools/cpp:toolchain", - "apple_platform_type": "ios", - }, - visibility = ["//visibility:public"], -) - -alias( - name = "apple", - actual = select({ - ":macos": ":macos", - ":ios": ":ios", - "//conditions:default": ":ios", # Arbitrarily chosen from above. - }), - visibility = ["//visibility:public"], -) - -config_setting( - name = "macos_i386", - values = { - "apple_platform_type": "macos", - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "macos_x86_64", - values = { - "apple_platform_type": "macos", - "cpu": "darwin_x86_64", - }, - visibility = ["//visibility:public"], -) - -[ - config_setting( - name = arch, - values = {"cpu": arch}, - visibility = ["//visibility:public"], - ) - for arch in [ - "ios_i386", - "ios_x86_64", - "ios_armv7", - "ios_arm64", - "ios_arm64e", - ] -] - -config_setting( - name = "windows", - values = {"cpu": "x64_windows"}, -) - -exports_files( - ["provisioning_profile.mobileprovision"], - visibility = ["//visibility:public"], -) diff --git a/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen b/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen deleted file mode 100644 index f3b74900c..000000000 --- a/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen +++ /dev/null @@ -1,137 +0,0 @@ -{ - "additionalFilePaths" : [ - "/BUILD", - "mediapipe/BUILD", - "mediapipe/examples/ios/common/BUILD", - "mediapipe/examples/ios/facedetectioncpu/BUILD", - "mediapipe/examples/ios/facedetectiongpu/BUILD", - "mediapipe/examples/ios/faceeffect/BUILD", - "mediapipe/examples/ios/facemeshgpu/BUILD", - "mediapipe/examples/ios/handdetectiongpu/BUILD", - "mediapipe/examples/ios/handtrackinggpu/BUILD", - "mediapipe/examples/ios/helloworld/BUILD", - "mediapipe/examples/ios/holistictrackinggpu/BUILD", - "mediapipe/examples/ios/iristrackinggpu/BUILD", - "mediapipe/examples/ios/objectdetectioncpu/BUILD", - "mediapipe/examples/ios/objectdetectiongpu/BUILD", - "mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD", - "mediapipe/examples/ios/posetrackinggpu/BUILD", - "mediapipe/examples/ios/selfiesegmentationgpu/BUILD", - "mediapipe/framework/BUILD", - "mediapipe/gpu/BUILD", - "mediapipe/objc/BUILD", - "mediapipe/objc/testing/app/BUILD" - ], - "buildTargets" : [ - "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp", - "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp", - "//mediapipe/examples/ios/faceeffect:FaceEffectApp", - "//mediapipe/examples/ios/facemeshgpu:FaceMeshGpuApp", - "//mediapipe/examples/ios/handdetectiongpu:HandDetectionGpuApp", - "//mediapipe/examples/ios/handtrackinggpu:HandTrackingGpuApp", - "//mediapipe/examples/ios/helloworld:HelloWorldApp", - "//mediapipe/examples/ios/holistictrackinggpu:HolisticTrackingGpuApp", - "//mediapipe/examples/ios/iristrackinggpu:IrisTrackingGpuApp", - "//mediapipe/examples/ios/objectdetectioncpu:ObjectDetectionCpuApp", - "//mediapipe/examples/ios/objectdetectiongpu:ObjectDetectionGpuApp", - "//mediapipe/examples/ios/objectdetectiontrackinggpu:ObjectDetectionTrackingGpuApp", - "//mediapipe/examples/ios/posetrackinggpu:PoseTrackingGpuApp", - "//mediapipe/examples/ios/selfiesegmentationgpu:SelfieSegmentationGpuApp", - "//mediapipe/objc:mediapipe_framework_ios" - ], - "optionSet" : { - "BazelBuildOptionsDebug" : { - "p" : "$(inherited)" - }, - "BazelBuildOptionsRelease" : { - "p" : "$(inherited)" - }, - "BazelBuildStartupOptionsDebug" : { - "p" : "$(inherited)" - }, - "BazelBuildStartupOptionsRelease" : { - "p" : "$(inherited)" - }, - "BuildActionPostActionScript" : { - "p" : "$(inherited)" - }, - "BuildActionPreActionScript" : { - "p" : "$(inherited)" - }, - "CommandlineArguments" : { - "p" : "$(inherited)" - }, - "EnvironmentVariables" : { - "p" : "$(inherited)" - }, - "LaunchActionPostActionScript" : { - "p" : "$(inherited)" - }, - "LaunchActionPreActionScript" : { - "p" : "$(inherited)" - }, - "ProjectGenerationBazelStartupOptions" : { - "p" : "$(inherited)" - }, - "TestActionPostActionScript" : { - "p" : "$(inherited)" - }, - "TestActionPreActionScript" : { - "p" : "$(inherited)" - } - }, - "projectName" : "Mediapipe", - "sourceFilters" : [ - "mediapipe", - "mediapipe/calculators", - "mediapipe/calculators/core", - "mediapipe/calculators/image", - "mediapipe/calculators/internal", - "mediapipe/calculators/tflite", - "mediapipe/calculators/util", - "mediapipe/examples", - "mediapipe/examples/ios", - "mediapipe/examples/ios/common", - "mediapipe/examples/ios/common/Base.lproj", - "mediapipe/examples/ios/facedetectioncpu", - "mediapipe/examples/ios/facedetectiongpu", - "mediapipe/examples/ios/faceeffect", - "mediapipe/examples/ios/faceeffect/Base.lproj", - "mediapipe/examples/ios/handdetectiongpu", - "mediapipe/examples/ios/handtrackinggpu", - "mediapipe/examples/ios/helloworld", - "mediapipe/examples/ios/holistictrackinggpu", - "mediapipe/examples/ios/iristrackinggpu", - "mediapipe/examples/ios/objectdetectioncpu", - "mediapipe/examples/ios/objectdetectiongpu", - "mediapipe/examples/ios/posetrackinggpu", - "mediapipe/examples/ios/selfiesegmentationgpu", - "mediapipe/framework", - "mediapipe/framework/deps", - "mediapipe/framework/formats", - "mediapipe/framework/formats/annotation", - "mediapipe/framework/formats/object_detection", - "mediapipe/framework/port", - "mediapipe/framework/profiler", - "mediapipe/framework/stream_handler", - "mediapipe/framework/tool", - "mediapipe/gpu", - "mediapipe/graphs", - "mediapipe/graphs/edge_detection", - "mediapipe/graphs/face_detection", - "mediapipe/graphs/face_geometry", - "mediapipe/graphs/hand_tracking", - "mediapipe/graphs/object_detection", - "mediapipe/graphs/pose_tracking", - "mediapipe/graphs/selfie_segmentation", - "mediapipe/models", - "mediapipe/modules", - "mediapipe/objc", - "mediapipe/util", - "mediapipe/util/android", - "mediapipe/util/android/file", - "mediapipe/util/android/file/base", - "mediapipe/util/tflite", - "mediapipe/util/tflite/operations" - ] -} diff --git a/mediapipe/MediaPipe.tulsiproj/project.tulsiconf b/mediapipe/MediaPipe.tulsiproj/project.tulsiconf deleted file mode 100644 index a2fe886cf..000000000 --- a/mediapipe/MediaPipe.tulsiproj/project.tulsiconf +++ /dev/null @@ -1,30 +0,0 @@ -{ - "configDefaults" : { - "optionSet" : { - "CLANG_CXX_LANGUAGE_STANDARD" : { - "p" : "c++14" - } - } - }, - "packages" : [ - "", - "mediapipe", - "mediapipe/examples/ios", - "mediapipe/examples/ios/facedetectioncpu", - "mediapipe/examples/ios/facedetectiongpu", - "mediapipe/examples/ios/faceeffect", - "mediapipe/examples/ios/facemeshgpu", - "mediapipe/examples/ios/handdetectiongpu", - "mediapipe/examples/ios/handtrackinggpu", - "mediapipe/examples/ios/holistictrackinggpu", - "mediapipe/examples/ios/iristrackinggpu", - "mediapipe/examples/ios/objectdetectioncpu", - "mediapipe/examples/ios/objectdetectiongpu", - "mediapipe/examples/ios/objectdetectiontrackinggpu", - "mediapipe/examples/ios/posetrackinggpu", - "mediapipe/examples/ios/selfiesegmentationgpu", - "mediapipe/objc" - ], - "projectName" : "Mediapipe", - "workspaceRoot" : "../.." -} diff --git a/mediapipe/__init__.py b/mediapipe/__init__.py deleted file mode 100644 index 69d7dfc6f..000000000 --- a/mediapipe/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Copyright 2019 - 2020 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD deleted file mode 100644 index ed6a509dc..000000000 --- a/mediapipe/calculators/audio/BUILD +++ /dev/null @@ -1,357 +0,0 @@ -# Copyright 2019, 2021 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -proto_library( - name = "mfcc_mel_calculators_proto", - srcs = ["mfcc_mel_calculators.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "mfcc_mel_calculators_cc_proto", - srcs = ["mfcc_mel_calculators.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":mfcc_mel_calculators_proto"], -) - -proto_library( - name = "rational_factor_resample_calculator_proto", - srcs = ["rational_factor_resample_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "rational_factor_resample_calculator_cc_proto", - srcs = ["rational_factor_resample_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":rational_factor_resample_calculator_proto"], -) - -proto_library( - name = "spectrogram_calculator_proto", - srcs = ["spectrogram_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "spectrogram_calculator_cc_proto", - srcs = ["spectrogram_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":spectrogram_calculator_proto"], -) - -proto_library( - name = "stabilized_log_calculator_proto", - srcs = ["stabilized_log_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "stabilized_log_calculator_cc_proto", - srcs = ["stabilized_log_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":stabilized_log_calculator_proto"], -) - -proto_library( - name = "time_series_framer_calculator_proto", - srcs = ["time_series_framer_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "time_series_framer_calculator_cc_proto", - srcs = ["time_series_framer_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":time_series_framer_calculator_proto"], -) - -cc_library( - name = "audio_decoder_calculator", - srcs = ["audio_decoder_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/util:audio_decoder", - "//mediapipe/util:audio_decoder_cc_proto", - ], - alwayslink = 1, -) - -cc_library( - name = "basic_time_series_calculators", - srcs = ["basic_time_series_calculators.cc"], - hdrs = ["basic_time_series_calculators.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:time_series_util", - "@com_google_absl//absl/strings", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "mfcc_mel_calculators", - srcs = ["mfcc_mel_calculators.cc"], - visibility = ["//visibility:public"], - deps = [ - ":mfcc_mel_calculators_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_util", - "@com_google_absl//absl/strings", - "@com_google_audio_tools//audio/dsp/mfcc", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "rational_factor_resample_calculator", - srcs = ["rational_factor_resample_calculator.cc"], - hdrs = ["rational_factor_resample_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":rational_factor_resample_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/util:time_series_util", - "@com_google_absl//absl/strings", - "@com_google_audio_tools//audio/dsp:resampler", - "@com_google_audio_tools//audio/dsp:resampler_q", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "stabilized_log_calculator", - srcs = ["stabilized_log_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":stabilized_log_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_util", - ], - alwayslink = 1, -) - -cc_library( - name = "spectrogram_calculator", - srcs = ["spectrogram_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":spectrogram_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_util", - "@com_google_absl//absl/strings", - "@com_google_audio_tools//audio/dsp:window_functions", - "@com_google_audio_tools//audio/dsp/spectrogram", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "time_series_framer_calculator", - srcs = ["time_series_framer_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":time_series_framer_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_util", - "@com_google_audio_tools//audio/dsp:window_functions", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_test( - name = "audio_decoder_calculator_test", - srcs = ["audio_decoder_calculator_test.cc"], - data = ["//mediapipe/calculators/audio/testdata:test_audios"], - deps = [ - ":audio_decoder_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/flags:flag", - ], -) - -cc_test( - name = "basic_time_series_calculators_test", - srcs = ["basic_time_series_calculators_test.cc"], - deps = [ - ":basic_time_series_calculators", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/util:time_series_test_util", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "mfcc_mel_calculators_test", - srcs = ["mfcc_mel_calculators_test.cc"], - deps = [ - ":mfcc_mel_calculators", - ":mfcc_mel_calculators_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_test_util", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "spectrogram_calculator_test", - srcs = ["spectrogram_calculator_test.cc"], - deps = [ - ":spectrogram_calculator", - ":spectrogram_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:benchmark", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_test_util", - "@com_google_audio_tools//audio/dsp:number_util", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "stabilized_log_calculator_test", - srcs = ["stabilized_log_calculator_test.cc"], - deps = [ - ":stabilized_log_calculator", - ":stabilized_log_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_test_util", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "time_series_framer_calculator_test", - srcs = ["time_series_framer_calculator_test.cc"], - deps = [ - ":time_series_framer_calculator", - ":time_series_framer_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:status", - "//mediapipe/util:time_series_test_util", - "@com_google_audio_tools//audio/dsp:window_functions", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "rational_factor_resample_calculator_test", - srcs = ["rational_factor_resample_calculator_test.cc"], - deps = [ - ":rational_factor_resample_calculator", - ":rational_factor_resample_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - "//mediapipe/util:time_series_test_util", - "@com_google_audio_tools//audio/dsp:signal_vector_util", - "@eigen_archive//:eigen3", - ], -) diff --git a/mediapipe/calculators/audio/audio_decoder_calculator.cc b/mediapipe/calculators/audio/audio_decoder_calculator.cc deleted file mode 100644 index 49c201b37..000000000 --- a/mediapipe/calculators/audio/audio_decoder_calculator.cc +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/audio_decoder.h" -#include "mediapipe/util/audio_decoder.pb.h" - -namespace mediapipe { - -// The AudioDecoderCalculator decodes an audio stream of the media file. It -// produces two output streams contain audio packets and the header infomation. -// -// Output Streams: -// AUDIO: Output audio frames (Matrix). -// AUDIO_HEADER: -// Optional audio header information output -// Input Side Packets: -// INPUT_FILE_PATH: The input file path. -// -// Example config: -// node { -// calculator: "AudioDecoderCalculator" -// input_side_packet: "INPUT_FILE_PATH:input_file_path" -// output_stream: "AUDIO:audio" -// output_stream: "AUDIO_HEADER:audio_header" -// node_options { -// [type.googleapis.com/mediapipe.AudioDecoderOptions]: { -// audio_stream { stream_index: 0 } -// start_time: 0 -// end_time: 1 -// } -// } -// -// TODO: support decoding multiple streams. -class AudioDecoderCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - std::unique_ptr decoder_; -}; - -absl::Status AudioDecoderCalculator::GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set(); - if (cc->InputSidePackets().HasTag("OPTIONS")) { - cc->InputSidePackets().Tag("OPTIONS").Set(); - } - cc->Outputs().Tag("AUDIO").Set(); - if (cc->Outputs().HasTag("AUDIO_HEADER")) { - cc->Outputs().Tag("AUDIO_HEADER").SetNone(); - } - return absl::OkStatus(); -} - -absl::Status AudioDecoderCalculator::Open(CalculatorContext* cc) { - const std::string& input_file_path = - cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get(); - const auto& decoder_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - decoder_ = absl::make_unique(); - MP_RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options)); - std::unique_ptr header = - absl::make_unique(); - if (decoder_->FillAudioHeader(decoder_options.audio_stream(0), header.get()) - .ok()) { - // Only pass on a header if the decoder could actually produce one. - // otherwise, the header will be empty. - cc->Outputs().Tag("AUDIO_HEADER").SetHeader(Adopt(header.release())); - } - cc->Outputs().Tag("AUDIO_HEADER").Close(); - return absl::OkStatus(); -} - -absl::Status AudioDecoderCalculator::Process(CalculatorContext* cc) { - Packet data; - int options_index = -1; - auto status = decoder_->GetData(&options_index, &data); - if (status.ok()) { - cc->Outputs().Tag("AUDIO").AddPacket(data); - } - return status; -} - -absl::Status AudioDecoderCalculator::Close(CalculatorContext* cc) { - return decoder_->Close(); -} - -REGISTER_CALCULATOR(AudioDecoderCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/audio_decoder_calculator_test.cc b/mediapipe/calculators/audio/audio_decoder_calculator_test.cc deleted file mode 100644 index 8e3babeb0..000000000 --- a/mediapipe/calculators/audio/audio_decoder_calculator_test.cc +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -TEST(AudioDecoderCalculatorTest, TestWAV) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "AudioDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "AUDIO:audio" - output_stream: "AUDIO_HEADER:audio_header" - node_options { - [type.googleapis.com/mediapipe.AudioDecoderOptions]: { - audio_stream { stream_index: 0 } - } - })pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/audio/" - "testdata/sine_wave_1k_44100_mono_2_sec_wav.audio")); - MP_ASSERT_OK(runner.Run()); - MP_EXPECT_OK(runner.Outputs() - .Tag("AUDIO_HEADER") - .header.ValidateAsType()); - const mediapipe::TimeSeriesHeader& header = - runner.Outputs() - .Tag("AUDIO_HEADER") - .header.Get(); - EXPECT_EQ(44100, header.sample_rate()); - EXPECT_EQ(1, header.num_channels()); - EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >= - std::ceil(44100.0 * 2 / 2048)); -} - -TEST(AudioDecoderCalculatorTest, Test48KWAV) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "AudioDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "AUDIO:audio" - output_stream: "AUDIO_HEADER:audio_header" - node_options { - [type.googleapis.com/mediapipe.AudioDecoderOptions]: { - audio_stream { stream_index: 0 } - } - })pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/audio/" - "testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio")); - MP_ASSERT_OK(runner.Run()); - MP_EXPECT_OK(runner.Outputs() - .Tag("AUDIO_HEADER") - .header.ValidateAsType()); - const mediapipe::TimeSeriesHeader& header = - runner.Outputs() - .Tag("AUDIO_HEADER") - .header.Get(); - EXPECT_EQ(48000, header.sample_rate()); - EXPECT_EQ(2, header.num_channels()); - EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >= - std::ceil(48000.0 * 2 / 1024)); -} - -TEST(AudioDecoderCalculatorTest, TestMP3) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "AudioDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "AUDIO:audio" - output_stream: "AUDIO_HEADER:audio_header" - node_options { - [type.googleapis.com/mediapipe.AudioDecoderOptions]: { - audio_stream { stream_index: 0 } - } - })pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/audio/" - "testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio")); - MP_ASSERT_OK(runner.Run()); - MP_EXPECT_OK(runner.Outputs() - .Tag("AUDIO_HEADER") - .header.ValidateAsType()); - const mediapipe::TimeSeriesHeader& header = - runner.Outputs() - .Tag("AUDIO_HEADER") - .header.Get(); - EXPECT_EQ(44100, header.sample_rate()); - EXPECT_EQ(2, header.num_channels()); - EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >= - std::ceil(44100.0 * 2 / 1152)); -} - -TEST(AudioDecoderCalculatorTest, TestAAC) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "AudioDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "AUDIO:audio" - output_stream: "AUDIO_HEADER:audio_header" - node_options { - [type.googleapis.com/mediapipe.AudioDecoderOptions]: { - audio_stream { stream_index: 0 } - } - })pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/audio/" - "testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio")); - MP_ASSERT_OK(runner.Run()); - MP_EXPECT_OK(runner.Outputs() - .Tag("AUDIO_HEADER") - .header.ValidateAsType()); - const mediapipe::TimeSeriesHeader& header = - runner.Outputs() - .Tag("AUDIO_HEADER") - .header.Get(); - EXPECT_EQ(44100, header.sample_rate()); - EXPECT_EQ(2, header.num_channels()); - EXPECT_TRUE(runner.Outputs().Tag("AUDIO").packets.size() >= - std::ceil(44100.0 * 2 / 1024)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/basic_time_series_calculators.cc b/mediapipe/calculators/audio/basic_time_series_calculators.cc deleted file mode 100644 index f7b24f6f6..000000000 --- a/mediapipe/calculators/audio/basic_time_series_calculators.cc +++ /dev/null @@ -1,405 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Basic Calculators that operate on TimeSeries streams. -#include "mediapipe/calculators/audio/basic_time_series_calculators.h" - -#include -#include - -#include "Eigen/Core" -#include "absl/strings/str_cat.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { -namespace { -static bool SafeMultiply(int x, int y, int* result) { - static_assert(sizeof(int64) >= 2 * sizeof(int), - "Unable to detect overflow after multiplication"); - const int64 big = static_cast(x) * static_cast(y); - if (big > static_cast(INT_MIN) && big < static_cast(INT_MAX)) { - if (result != nullptr) *result = static_cast(big); - return true; - } else { - return false; - } -} -} // namespace - -absl::Status BasicTimeSeriesCalculatorBase::GetContract( - CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Input stream with TimeSeriesHeader. - ); - cc->Outputs().Index(0).Set( - // Output stream with TimeSeriesHeader. - ); - return absl::OkStatus(); -} - -absl::Status BasicTimeSeriesCalculatorBase::Open(CalculatorContext* cc) { - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - - auto output_header = new TimeSeriesHeader(input_header); - MP_RETURN_IF_ERROR(MutateHeader(output_header)); - cc->Outputs().Index(0).SetHeader(Adopt(output_header)); - - cc->SetOffset(0); - - return absl::OkStatus(); -} - -absl::Status BasicTimeSeriesCalculatorBase::Process(CalculatorContext* cc) { - const Matrix& input = cc->Inputs().Index(0).Get(); - MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader( - input, cc->Inputs().Index(0).Header().Get())); - - std::unique_ptr output(new Matrix(ProcessMatrix(input))); - MP_RETURN_IF_ERROR(time_series_util::IsMatrixShapeConsistentWithHeader( - *output, cc->Outputs().Index(0).Header().Get())); - - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -absl::Status BasicTimeSeriesCalculatorBase::MutateHeader( - TimeSeriesHeader* output_header) { - return absl::OkStatus(); -} - -// Calculator to sum an input time series across channels. This is -// useful for e.g. computing 'summary SAI' pitchogram features. -// -// Options proto: None. -class SumTimeSeriesAcrossChannelsCalculator - : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_channels(1); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.colwise().sum(); - } -}; -REGISTER_CALCULATOR(SumTimeSeriesAcrossChannelsCalculator); - -// Calculator to average an input time series across channels. This is -// useful for e.g. converting stereo or multi-channel files to mono. -// -// Options proto: None. -class AverageTimeSeriesAcrossChannelsCalculator - : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_channels(1); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.colwise().mean(); - } -}; -REGISTER_CALCULATOR(AverageTimeSeriesAcrossChannelsCalculator); - -// Calculator to convert a (temporal) summary SAI stream (a single-channel -// stream output by SumTimeSeriesAcrossChannelsCalculator) into pitchogram -// frames by transposing the input packets, swapping the time and channel axes. -// -// Options proto: None. -class SummarySaiToPitchogramCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - if (output_header->num_channels() != 1) { - return tool::StatusInvalid( - absl::StrCat("Expected single-channel input, got ", - output_header->num_channels())); - } - output_header->set_num_channels(output_header->num_samples()); - output_header->set_num_samples(1); - output_header->set_sample_rate(output_header->packet_rate()); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.transpose(); - } -}; -REGISTER_CALCULATOR(SummarySaiToPitchogramCalculator); - -// Calculator to reverse the order of channels in TimeSeries packets. -// This is useful for e.g. interfacing with the speech pipeline which uses the -// opposite convention to the hearing filterbanks. -// -// Options proto: None. -class ReverseChannelOrderCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.colwise().reverse(); - } -}; -REGISTER_CALCULATOR(ReverseChannelOrderCalculator); - -// Calculator to flatten all samples in a TimeSeries packet down into -// a single 'sample' vector. This is useful for e.g. stacking several -// frames of features into a single feature vector. -// -// Options proto: None. -class FlattenPacketCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - const int num_input_channels = output_header->num_channels(); - const int num_input_samples = output_header->num_samples(); - RET_CHECK(num_input_channels >= 0) - << "FlattenPacketCalculator: num_input_channels < 0"; - RET_CHECK(num_input_samples >= 0) - << "FlattenPacketCalculator: num_input_samples < 0"; - int output_num_channels; - RET_CHECK(SafeMultiply(num_input_channels, num_input_samples, - &output_num_channels)) - << "FlattenPacketCalculator: Multiplication failed."; - output_header->set_num_channels(output_num_channels); - output_header->set_num_samples(1); - output_header->set_sample_rate(output_header->packet_rate()); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - // Flatten by interleaving channels so that full samples are - // stacked on top of each other instead of interleaving samples - // from the same channel. - Matrix output(input_matrix.size(), 1); - for (int sample = 0; sample < input_matrix.cols(); ++sample) { - output.middleRows(sample * input_matrix.rows(), input_matrix.rows()) = - input_matrix.col(sample); - } - return output; - } -}; -REGISTER_CALCULATOR(FlattenPacketCalculator); - -// Calculator to subtract the within-packet mean for each channel from each -// corresponding channel. -// -// Options proto: None. -class SubtractMeanCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - Matrix mean = input_matrix.rowwise().mean(); - return input_matrix - mean.replicate(1, input_matrix.cols()); - } -}; -REGISTER_CALCULATOR(SubtractMeanCalculator); - -// Calculator to subtract the mean over all values (across all times and -// channels) in a Packet from the values in that Packet. -// -// Options proto: None. -class SubtractMeanAcrossChannelsCalculator - : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - auto mean = input_matrix.mean(); - return (input_matrix.array() - mean).matrix(); - } -}; -REGISTER_CALCULATOR(SubtractMeanAcrossChannelsCalculator); - -// Calculator to divide all values in a Packet by the average value across all -// times and channels in the packet. This is useful for normalizing -// nonnegative quantities like power, but might cause unexpected results if used -// with Packets that can contain negative numbers. -// -// If mean is exactly zero, the output will be a matrix of all ones, because -// that's what happens in other cases where all values are equal. -// -// Options proto: None. -class DivideByMeanAcrossChannelsCalculator - : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - auto mean = input_matrix.mean(); - - if (mean != 0) { - return input_matrix / mean; - - // When used with nonnegative matrices, the mean will only be zero if the - // entire matrix is exactly zero. If mean is exactly zero, the output will - // be a matrix of all ones, because that's what happens in other cases - // where - // all values are equal. - } else { - return Matrix::Ones(input_matrix.rows(), input_matrix.cols()); - } - } -}; -REGISTER_CALCULATOR(DivideByMeanAcrossChannelsCalculator); - -// Calculator to calculate the mean for each channel. -// -// Options proto: None. -class MeanCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_samples(1); - output_header->set_sample_rate(output_header->packet_rate()); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.rowwise().mean(); - } -}; -REGISTER_CALCULATOR(MeanCalculator); - -// Calculator to calculate the uncorrected sample standard deviation in each -// channel, independently for each Packet. I.e. divide by the number of samples -// in the Packet, not ( - 1). -// -// Options proto: None. -class StandardDeviationCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_samples(1); - output_header->set_sample_rate(output_header->packet_rate()); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - Eigen::VectorXf mean = input_matrix.rowwise().mean(); - return (input_matrix.colwise() - mean).rowwise().norm() / - sqrt(input_matrix.cols()); - } -}; -REGISTER_CALCULATOR(StandardDeviationCalculator); - -// Calculator to calculate the covariance matrix. If the input matrix -// has N channels, the output matrix will be an N by N symmetric -// matrix. -// -// Options proto: None. -class CovarianceCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_samples(output_header->num_channels()); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - auto mean = input_matrix.rowwise().mean(); - auto zero_mean_input = - input_matrix - mean.replicate(1, input_matrix.cols()); - return (zero_mean_input * zero_mean_input.transpose()) / - input_matrix.cols(); - } -}; -REGISTER_CALCULATOR(CovarianceCalculator); - -// Calculator to get the per column L2 norm of an input time series. -// -// Options proto: None. -class L2NormCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - output_header->set_num_channels(1); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.colwise().norm(); - } -}; -REGISTER_CALCULATOR(L2NormCalculator); - -// Calculator to convert each column of a matrix to a unit vector. -// -// Options proto: None. -class L2NormalizeColumnCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.colwise().normalized(); - } -}; -REGISTER_CALCULATOR(L2NormalizeColumnCalculator); - -// Calculator to apply L2 normalization to the input matrix. -// -// Returns the matrix as is if the RMS is <= 1E-8. -// Options proto: None. -class L2NormalizeCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - constexpr double kEpsilon = 1e-8; - double rms = std::sqrt(input_matrix.array().square().mean()); - if (rms <= kEpsilon) { - return input_matrix; - } - return input_matrix / rms; - } -}; -REGISTER_CALCULATOR(L2NormalizeCalculator); - -// Calculator to apply Peak normalization to the input matrix. -// -// Returns the matrix as is if the peak is <= 1E-8. -// Options proto: None. -class PeakNormalizeCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - constexpr double kEpsilon = 1e-8; - double max_pcm = input_matrix.cwiseAbs().maxCoeff(); - if (max_pcm <= kEpsilon) { - return input_matrix; - } - return input_matrix / max_pcm; - } -}; -REGISTER_CALCULATOR(PeakNormalizeCalculator); - -// Calculator to compute the elementwise square of an input time series. -// -// Options proto: None. -class ElementwiseSquareCalculator : public BasicTimeSeriesCalculatorBase { - protected: - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.array().square(); - } -}; -REGISTER_CALCULATOR(ElementwiseSquareCalculator); - -// Calculator that outputs first floor(num_samples / 2) of the samples. -// -// Options proto: None. -class FirstHalfSlicerCalculator : public BasicTimeSeriesCalculatorBase { - protected: - absl::Status MutateHeader(TimeSeriesHeader* output_header) final { - const int num_input_samples = output_header->num_samples(); - RET_CHECK(num_input_samples >= 0) - << "FirstHalfSlicerCalculator: num_input_samples < 0"; - output_header->set_num_samples(num_input_samples / 2); - return absl::OkStatus(); - } - - Matrix ProcessMatrix(const Matrix& input_matrix) final { - return input_matrix.block(0, 0, input_matrix.rows(), - input_matrix.cols() / 2); - } -}; -REGISTER_CALCULATOR(FirstHalfSlicerCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/basic_time_series_calculators.h b/mediapipe/calculators/audio/basic_time_series_calculators.h deleted file mode 100644 index ef31f3448..000000000 --- a/mediapipe/calculators/audio/basic_time_series_calculators.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Abstract base class for basic MediaPipe calculators that operate on -// TimeSeries streams and don't require any Options protos. -// Subclasses must override ProcessMatrix, and optionally -// MutateHeader. - -#ifndef MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_ -#define MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_ - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" - -namespace mediapipe { - -class BasicTimeSeriesCalculatorBase : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) final; - absl::Status Process(CalculatorContext* cc) final; - - protected: - // Open() calls this method to mutate the output stream header. The input - // to this function will contain a copy of the input stream header, so - // subclasses that do not need to mutate the header do not need to override - // it. - virtual absl::Status MutateHeader(TimeSeriesHeader* output_header); - - // Process() calls this method on each packet to compute the output matrix. - virtual Matrix ProcessMatrix(const Matrix& input_matrix) = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_AUDIO_BASIC_TIME_SERIES_CALCULATORS_H_ diff --git a/mediapipe/calculators/audio/basic_time_series_calculators_test.cc b/mediapipe/calculators/audio/basic_time_series_calculators_test.cc deleted file mode 100644 index 7211b83fe..000000000 --- a/mediapipe/calculators/audio/basic_time_series_calculators_test.cc +++ /dev/null @@ -1,515 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "Eigen/Core" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { - -class SumTimeSeriesAcrossChannelsCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { - calculator_name_ = "SumTimeSeriesAcrossChannelsCalculator"; - } -}; - -TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, IsNoOpOnSingleChannelInputs) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 1 num_samples: 5"); - const Matrix input = - Matrix::Random(header.num_channels(), header.num_samples()); - - Test(header, {input}, header, {input}); -} - -TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 3 num_samples: 5"); - TimeSeriesHeader output_header(header); - output_header.set_num_channels(1); - - Test(header, - {Matrix::Constant(header.num_channels(), header.num_samples(), 1)}, - output_header, - {Matrix::Constant(1, header.num_samples(), header.num_channels())}); -} - -TEST_F(SumTimeSeriesAcrossChannelsCalculatorTest, MultiplePackets) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 3 num_samples: 5"); - Matrix in(header.num_channels(), header.num_samples()); - in << 10, -1, -1, 0, 0, 20, -2, 0, 1, 0, 30, -3, 1, 0, 12; - - TimeSeriesHeader output_header(header); - output_header.set_num_channels(1); - Matrix out(1, header.num_samples()); - out << 60, -6, 0, 1, 12; - - Test(header, {in, 2 * in, in + Matrix::Constant(in.rows(), in.cols(), 3.5f)}, - output_header, - {out, 2 * out, - out + Matrix::Constant(out.rows(), out.cols(), - 3.5 * header.num_channels())}); -} - -class AverageTimeSeriesAcrossChannelsCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { - calculator_name_ = "AverageTimeSeriesAcrossChannelsCalculator"; - } -}; - -TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest, - IsNoOpOnSingleChannelInputs) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 1 num_samples: 5"); - const Matrix input = - Matrix::Random(header.num_channels(), header.num_samples()); - - Test(header, {input}, header, {input}); -} - -TEST_F(AverageTimeSeriesAcrossChannelsCalculatorTest, ConstantPacket) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 3 num_samples: 5"); - TimeSeriesHeader output_header(header); - output_header.set_num_channels(1); - - Matrix input = - Matrix::Constant(header.num_channels(), header.num_samples(), 0.0); - input.row(0) = Matrix::Constant(1, header.num_samples(), 1.0); - - Test( - header, {input}, output_header, - {Matrix::Constant(1, header.num_samples(), 1.0 / header.num_channels())}); -} - -class SummarySaiToPitchogramCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { - calculator_name_ = "SummarySaiToPitchogramCalculator"; - } -}; - -TEST_F(SummarySaiToPitchogramCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 1 num_samples: 3"); - Matrix input(1, input_header.num_samples()); - input << 3, -9, 4; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 5.0 packet_rate: 5.0 num_channels: 3 num_samples: 1"); - Matrix output(input_header.num_samples(), 1); - output << 3, -9, 4; - - Test(input_header, {input}, output_header, {output}); -} - -class ReverseChannelOrderCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "ReverseChannelOrderCalculator"; } -}; - -TEST_F(ReverseChannelOrderCalculatorTest, IsNoOpOnSingleChannelInputs) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 1 num_samples: 5"); - const Matrix input = - Matrix::Random(header.num_channels(), header.num_samples()); - - Test(header, {input}, header, {input}); -} - -TEST_F(ReverseChannelOrderCalculatorTest, SinglePacket) { - const TimeSeriesHeader header = ParseTextProtoOrDie( - "sample_rate: 8000.0 num_channels: 5 num_samples: 2"); - Matrix input(header.num_channels(), header.num_samples()); - input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5; - Matrix output(header.num_channels(), header.num_samples()); - output.transpose() << 5, 4, 3, 2, 1, -5, -4, -3, -2, -1; - - Test(header, {input}, header, {output}); -} - -class FlattenPacketCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "FlattenPacketCalculator"; } -}; - -TEST_F(FlattenPacketCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input.transpose() << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5; - Matrix output(10, 1); - output << 1, 2, 3, 4, 5, -1, -2, -3, -4, -5; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 10.0 packet_rate: 10.0 num_channels: 10 num_samples: 1"); - Test(input_header, {input}, output_header, {output}); -} - -class SubtractMeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "SubtractMeanCalculator"; } -}; - -TEST_F(SubtractMeanCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - Matrix output(input_header.num_channels(), input_header.num_samples()); - - // clang-format off - input.transpose() << 1, 0, 3, 0, 1, - -1, -2, -3, 4, 7; - output.transpose() << 1, 1, 3, -2, -3, - -1, -1, -3, 2, 3; - // clang-format on - - const TimeSeriesHeader output_header = input_header; - Test(input_header, {input}, output_header, {output}); -} - -class SubtractMeanAcrossChannelsCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { - calculator_name_ = "SubtractMeanAcrossChannelsCalculator"; - } -}; - -TEST_F(SubtractMeanAcrossChannelsCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - TimeSeriesHeader output_header(input_header); - output_header.set_num_samples(2); - - Matrix input(input_header.num_channels(), input_header.num_samples()); - Matrix output(output_header.num_channels(), output_header.num_samples()); - - // clang-format off - input.transpose() << 1.0, 2.0, 3.0, - 4.0, 5.0, 6.0; - output.transpose() << 1.0 - 3.5, 2.0 - 3.5, 3.0 - 3.5, - 4.0 - 3.5, 5.0 - 3.5, 6.0 - 3.5; - // clang-format on - - Test(input_header, {input}, output_header, {output}); -} - -class DivideByMeanAcrossChannelsCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { - calculator_name_ = "DivideByMeanAcrossChannelsCalculator"; - } -}; - -TEST_F(DivideByMeanAcrossChannelsCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0; - - TimeSeriesHeader output_header(input_header); - output_header.set_num_samples(2); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output.transpose() << 1.0 / 3.5, 2.0 / 3.5, 3.0 / 3.5, 4.0 / 3.5, 5.0 / 3.5, - 6.0 / 3.5; - - Test(input_header, {input}, output_header, {output}); -} - -TEST_F(DivideByMeanAcrossChannelsCalculatorTest, ReturnsOneForZeroMean) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input.transpose() << -3.0, -2.0, -1.0, 1.0, 2.0, 3.0; - - TimeSeriesHeader output_header(input_header); - output_header.set_num_samples(2); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output.transpose() << 1.0, 1.0, 1.0, 1.0, 1.0, 1.0; - - Test(input_header, {input}, output_header, {output}); -} - -class MeanCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "MeanCalculator"; } -}; - -TEST_F(MeanCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input.transpose() << 1.0, 2.0, 3.0, 4.0, 5.0, 6.0; - - TimeSeriesHeader output_header(input_header); - output_header.set_num_samples(1); - output_header.set_sample_rate(10.0); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << (1.0 + 4.0) / 2, (2.0 + 5.0) / 2, (3.0 + 6.0) / 2; - - Test(input_header, {input}, output_header, {output}); -} - -class StandardDeviationCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "StandardDeviationCalculator"; } -}; - -TEST_F(StandardDeviationCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input.transpose() << 0.0, 2.0, 3.0, 4.0, 5.0, 8.0; - - TimeSeriesHeader output_header(input_header); - output_header.set_sample_rate(10.0); - output_header.set_num_samples(1); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << sqrt((pow(0.0 - 2.0, 2) + pow(4.0 - 2.0, 2)) / 2), - sqrt((pow(2.0 - 3.5, 2) + pow(5.0 - 3.5, 2)) / 2), - sqrt((pow(3.0 - 5.5, 2) + pow(8.0 - 5.5, 2)) / 2); - - Test(input_header, {input}, output_header, {output}); -} - -class CovarianceCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "CovarianceCalculator"; } -}; - -TEST_F(CovarianceCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 3 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - - // We'll specify in transposed form so we can write one channel at a time. - input << 1.0, 3.0, 5.0, 9.0, -1.0, -3.0; - - TimeSeriesHeader output_header(input_header); - output_header.set_num_samples(output_header.num_channels()); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << 1, 2, -1, 2, 4, -2, -1, -2, 1; - Test(input_header, {input}, output_header, {output}); -} - -class L2NormCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "L2NormCalculator"; } -}; - -TEST_F(L2NormCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 3, 5, 8, 4, 12, -15; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 1 num_samples: 3"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << 5, 13, 17; - - Test(input_header, {input}, output_header, {output}); -} - -class L2NormalizeColumnCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "L2NormalizeColumnCalculator"; } -}; - -TEST_F(L2NormalizeColumnCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - - // The values in output are column-wise L2 normalized - // e.g. - // |a| -> |a/sqrt(a^2 + b^2)| - // |b| |b/sqrt(a^2 + b^2)| - output << 0.51449579000473022, 0.40613847970962524, 0.70710676908493042, - 0.85749292373657227, 0.91381156444549561, 0.70710676908493042; - - Test(input_header, {input}, output_header, {output}); -} - -class L2NormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "L2NormalizeCalculator"; } -}; - -TEST_F(L2NormalizeCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - - // The values in output are L2 normalized - // a -> a/sqrt(a^2 + b^2 + c^2 + ...) * sqrt(matrix.cols()*matrix.rows()) - output << 0.45661166, 0.60881555, 1.21763109, 0.76101943, 1.36983498, - 1.21763109; - - Test(input_header, {input}, output_header, {output}); -} - -TEST_F(L2NormalizeCalculatorTest, UnitMatrixStaysUnchanged) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 3 num_samples: 5"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, - 1.0, -1.0, 1.0; - - Test(input_header, {input}, input_header, {input}); -} - -class PeakNormalizeCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "PeakNormalizeCalculator"; } -}; - -TEST_F(PeakNormalizeCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 0.3, 0.4, 0.8, 0.5, 0.9, 0.8; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << 0.33333333, 0.44444444, 0.88888889, 0.55555556, 1.0, 0.88888889; - - Test(input_header, {input}, output_header, {output}); -} - -TEST_F(PeakNormalizeCalculatorTest, UnitMatrixStaysUnchanged) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 3 num_samples: 5"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, 1.0, -1.0, - 1.0, -1.0, 1.0; - - Test(input_header, {input}, input_header, {input}); -} - -class ElementwiseSquareCalculatorTest - : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "ElementwiseSquareCalculator"; } -}; - -TEST_F(ElementwiseSquareCalculatorTest, SinglePacket) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - input << 3, 5, 8, 4, 12, -15; - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 8000.0 packet_rate: 5.0 num_channels: 2 num_samples: 3"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output << 9, 25, 64, 16, 144, 225; - - Test(input_header, {input}, output_header, {output}); -} - -class FirstHalfSlicerCalculatorTest : public BasicTimeSeriesCalculatorTestBase { - protected: - void SetUp() override { calculator_name_ = "FirstHalfSlicerCalculator"; } -}; - -TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketEvenNumSamples) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - // clang-format off - input.transpose() << 0, 1, 2, 3, 4, - 5, 6, 7, 8, 9; - // clang-format on - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output.transpose() << 0, 1, 2, 3, 4; - - Test(input_header, {input}, output_header, {output}); -} - -TEST_F(FirstHalfSlicerCalculatorTest, SinglePacketOddNumSamples) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 3"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - // clang-format off - input.transpose() << 0, 1, 2, 3, 4, - 5, 6, 7, 8, 9, - 0, 0, 0, 0, 0; - // clang-format on - - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output.transpose() << 0, 1, 2, 3, 4; - - Test(input_header, {input}, output_header, {output}); -} - -TEST_F(FirstHalfSlicerCalculatorTest, MultiplePackets) { - const TimeSeriesHeader input_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 2"); - Matrix input(input_header.num_channels(), input_header.num_samples()); - // clang-format off - input.transpose() << 0, 1, 2, 3, 4, - 5, 6, 7, 8, 9; - // clang-format on - const TimeSeriesHeader output_header = ParseTextProtoOrDie( - "sample_rate: 20.0 packet_rate: 10.0 num_channels: 5 num_samples: 1"); - Matrix output(output_header.num_channels(), output_header.num_samples()); - output.transpose() << 0, 1, 2, 3, 4; - - Test(input_header, - {input, 2 * input, - input + Matrix::Constant(input.rows(), input.cols(), 3.5f)}, - output_header, - {output, 2 * output, - output + Matrix::Constant(output.rows(), output.cols(), 3.5f)}); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators.cc b/mediapipe/calculators/audio/mfcc_mel_calculators.cc deleted file mode 100644 index a63b9d6ea..000000000 --- a/mediapipe/calculators/audio/mfcc_mel_calculators.cc +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// MediaPipe Calculator wrapper around audio/dsp/mfcc/ -// classes MelFilterbank (magnitude spectrograms warped to the Mel -// approximation of the auditory frequency scale) and Mfcc (Mel Frequency -// Cepstral Coefficients, the decorrelated transform of log-Mel-spectrum -// commonly used as acoustic features in speech and other audio tasks. -// Both calculators expect as input the SQUARED_MAGNITUDE-domain outputs -// from the MediaPipe SpectrogramCalculator object. -#include -#include - -#include "Eigen/Core" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "audio/dsp/mfcc/mel_filterbank.h" -#include "audio/dsp/mfcc/mfcc.h" -#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { - -namespace { - -// Portable version of TimeSeriesHeader's DebugString. -std::string PortableDebugString(const TimeSeriesHeader& header) { - std::string unsubstituted_header_debug_str = R"( - sample_rate: $0 - num_channels: $1 - num_samples: $2 - packet_rate: $3 - audio_sample_rate: $4 - )"; - return absl::Substitute(unsubstituted_header_debug_str, header.sample_rate(), - header.num_channels(), header.num_samples(), - header.packet_rate(), header.audio_sample_rate()); -} - -} // namespace - -// Abstract base class for Calculators that transform feature vectors on a -// frame-by-frame basis. -// Subclasses must override pure virtual methods ConfigureTransform and -// TransformFrame. -// Input and output MediaPipe packets are matrices with one column per frame, -// and one row per feature dimension. Each input packet results in an -// output packet with the same number of columns (but differing numbers of -// rows corresponding to the new feature space). -class FramewiseTransformCalculatorBase : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Sequence of Matrices, each column describing a particular time frame, - // each row a feature dimension, with TimeSeriesHeader. - ); - cc->Outputs().Index(0).Set( - // Sequence of Matrices, each column describing a particular time frame, - // each row a feature dimension, with TimeSeriesHeader. - ); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final; - absl::Status Process(CalculatorContext* cc) final; - - int num_output_channels(void) { return num_output_channels_; } - - void set_num_output_channels(int num_output_channels) { - num_output_channels_ = num_output_channels; - } - - private: - // Takes header and options, and sets up state including calling - // set_num_output_channels() on the base object. - virtual absl::Status ConfigureTransform(const TimeSeriesHeader& header, - CalculatorContext* cc) = 0; - - // Takes a vector corresponding to an input frame, and - // perform the specific transformation to produce an output frame. - virtual void TransformFrame(const std::vector& input, - std::vector* output) const = 0; - - private: - int num_output_channels_; -}; - -absl::Status FramewiseTransformCalculatorBase::Open(CalculatorContext* cc) { - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - - absl::Status status = ConfigureTransform(input_header, cc); - - auto output_header = new TimeSeriesHeader(input_header); - output_header->set_num_channels(num_output_channels_); - cc->Outputs().Index(0).SetHeader(Adopt(output_header)); - - cc->SetOffset(0); - - return status; -} - -absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) { - const Matrix& input = cc->Inputs().Index(0).Get(); - const int num_frames = input.cols(); - std::unique_ptr output(new Matrix(num_output_channels_, num_frames)); - // The main work here is converting each column of the float Matrix - // into a vector of doubles, which is what our target functions from - // dsp_core consume, and doing the reverse with their output. - std::vector input_frame(input.rows()); - std::vector output_frame(num_output_channels_); - - for (int frame = 0; frame < num_frames; ++frame) { - // Copy input from Eigen::Matrix column to vector. - Eigen::Map input_frame_map(&input_frame[0], - input_frame.size(), 1); - input_frame_map = input.col(frame).cast(); - - // Perform the actual transformation. - TransformFrame(input_frame, &output_frame); - - // Copy output from vector to Eigen::Vector. - CHECK_EQ(output_frame.size(), num_output_channels_); - Eigen::Map output_frame_map(&output_frame[0], - output_frame.size(), 1); - output->col(frame) = output_frame_map.cast(); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -// Calculator wrapper around the dsp/mfcc/mfcc.cc routine. -// Take frames of squared-magnitude spectra from the SpectrogramCalculator -// and convert them into Mel Frequency Cepstral Coefficients. -// -// Example config: -// node { -// calculator: "MfccCalculator" -// input_stream: "spectrogram_frames_stream" -// output_stream: "mfcc_frames_stream" -// options { -// [mediapipe.MfccCalculatorOptions.ext] { -// mel_spectrum_params { -// channel_count: 20 -// min_frequency_hertz: 125.0 -// max_frequency_hertz: 3800.0 -// } -// mfcc_count: 13 -// } -// } -// } -class MfccCalculator : public FramewiseTransformCalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - return FramewiseTransformCalculatorBase::GetContract(cc); - } - - private: - absl::Status ConfigureTransform(const TimeSeriesHeader& header, - CalculatorContext* cc) override { - MfccCalculatorOptions mfcc_options = cc->Options(); - mfcc_.reset(new audio_dsp::Mfcc()); - int input_length = header.num_channels(); - // Set up the parameters to the Mfcc object. - set_num_output_channels(mfcc_options.mfcc_count()); - mfcc_->set_dct_coefficient_count(num_output_channels()); - mfcc_->set_upper_frequency_limit( - mfcc_options.mel_spectrum_params().max_frequency_hertz()); - mfcc_->set_lower_frequency_limit( - mfcc_options.mel_spectrum_params().min_frequency_hertz()); - mfcc_->set_filterbank_channel_count( - mfcc_options.mel_spectrum_params().channel_count()); - // An upstream calculator (such as SpectrogramCalculator) must store - // the sample rate of its input audio waveform in the TimeSeries Header. - // audio_dsp::MelFilterBank needs to know this to - // correctly interpret the spectrogram bins. - if (!header.has_audio_sample_rate()) { - return absl::InvalidArgumentError( - absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ", - PortableDebugString(header))); - } - // Now we can initialize the Mfcc object. - bool initialized = - mfcc_->Initialize(input_length, header.audio_sample_rate()); - - if (initialized) { - return absl::OkStatus(); - } else { - return absl::Status(absl::StatusCode::kInternal, - "Mfcc::Initialize returned uninitialized"); - } - } - - void TransformFrame(const std::vector& input, - std::vector* output) const override { - mfcc_->Compute(input, output); - } - - private: - std::unique_ptr mfcc_; -}; -REGISTER_CALCULATOR(MfccCalculator); - -// Calculator wrapper around the dsp/mfcc/mel_filterbank.cc routine. -// Take frames of squared-magnitude spectra from the SpectrogramCalculator -// and convert them into Mel-warped (linear-magnitude) spectra. -// Note: This code computes a mel-frequency filterbank, using a simple -// algorithm that gives bad results (some mel channels that are always zero) -// if you ask for too many channels. -class MelSpectrumCalculator : public FramewiseTransformCalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - return FramewiseTransformCalculatorBase::GetContract(cc); - } - - private: - absl::Status ConfigureTransform(const TimeSeriesHeader& header, - CalculatorContext* cc) override { - MelSpectrumCalculatorOptions mel_spectrum_options = - cc->Options(); - mel_filterbank_.reset(new audio_dsp::MelFilterbank()); - int input_length = header.num_channels(); - set_num_output_channels(mel_spectrum_options.channel_count()); - // An upstream calculator (such as SpectrogramCalculator) must store - // the sample rate of its input audio waveform in the TimeSeries Header. - // audio_dsp::MelFilterBank needs to know this to - // correctly interpret the spectrogram bins. - if (!header.has_audio_sample_rate()) { - return absl::InvalidArgumentError( - absl::StrCat("No audio_sample_rate in input TimeSeriesHeader ", - PortableDebugString(header))); - } - bool initialized = mel_filterbank_->Initialize( - input_length, header.audio_sample_rate(), num_output_channels(), - mel_spectrum_options.min_frequency_hertz(), - mel_spectrum_options.max_frequency_hertz()); - - if (initialized) { - return absl::OkStatus(); - } else { - return absl::Status(absl::StatusCode::kInternal, - "mfcc::Initialize returned uninitialized"); - } - } - - void TransformFrame(const std::vector& input, - std::vector* output) const override { - mel_filterbank_->Compute(input, output); - } - - private: - std::unique_ptr mel_filterbank_; -}; -REGISTER_CALCULATOR(MelSpectrumCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators.proto b/mediapipe/calculators/audio/mfcc_mel_calculators.proto deleted file mode 100644 index 89af5eb41..000000000 --- a/mediapipe/calculators/audio/mfcc_mel_calculators.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message MelSpectrumCalculatorOptions { - extend CalculatorOptions { - optional MelSpectrumCalculatorOptions ext = 78581812; - } - // The fields are to populate the config parameters in - // audio/dsp/mfcc/mel_filterbank.h - // but the names are chose to mirror - // audio/hearing/filterbanks/cochlea_gammatone_filterbank.proto - // and the default values match those in - // speech/greco3/frontend/filter_bank.proto . - - // Total number of frequency bands to use. - optional int32 channel_count = 1 [default = 20]; - // Lower edge of lowest triangular Mel band. - optional float min_frequency_hertz = 2 [default = 125.0]; - // Upper edge of highest triangular Mel band. - optional float max_frequency_hertz = 3 [default = 3800.0]; -} - -message MfccCalculatorOptions { - extend CalculatorOptions { - optional MfccCalculatorOptions ext = 78450441; - } - - // Specification of the underlying mel filterbank. - optional MelSpectrumCalculatorOptions mel_spectrum_params = 1; - - // How many MFCC coefficients to emit. - optional uint32 mfcc_count = 2 [default = 13]; -} diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators_test.cc b/mediapipe/calculators/audio/mfcc_mel_calculators_test.cc deleted file mode 100644 index e7e312db9..000000000 --- a/mediapipe/calculators/audio/mfcc_mel_calculators_test.cc +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include - -#include "Eigen/Core" -#include "mediapipe/calculators/audio/mfcc_mel_calculators.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { - -// Use a sample rate that is unlikely to be a default somewhere. -const float kAudioSampleRate = 8800.0; - -template -class FramewiseTransformCalculatorTest - : public TimeSeriesCalculatorTest { - protected: - void SetUp() override { - this->calculator_name_ = CalculatorName; - this->num_input_channels_ = 129; - // This is the frame rate coming out of the SpectrogramCalculator. - this->input_sample_rate_ = 100.0; - } - - // Returns the number of samples per packet. - int GenerateRandomNonnegInputStream(int num_packets) { - const double kSecondsPerPacket = 0.2; - const int num_samples_per_packet = - kSecondsPerPacket * this->input_sample_rate_; - for (int i = 0; i < num_packets; ++i) { - const int timestamp = - i * kSecondsPerPacket * Timestamp::kTimestampUnitsPerSecond; - // Mfcc, MelSpectrum expect squared-magnitude inputs, so make - // sure the input data has no negative values. - Matrix* sqdata = this->NewRandomMatrix(this->num_input_channels_, - num_samples_per_packet); - *sqdata = sqdata->array().square(); - this->AppendInputPacket(sqdata, timestamp); - } - return num_samples_per_packet; - } - - void CheckOutputPacketMetadata(int expected_num_channels, - int expected_num_samples_per_packet) { - int expected_timestamp = 0; - for (const auto& packet : this->output().packets) { - EXPECT_EQ(expected_timestamp, packet.Timestamp().Value()); - expected_timestamp += expected_num_samples_per_packet / - this->input_sample_rate_ * - Timestamp::kTimestampUnitsPerSecond; - - const Matrix& output_matrix = packet.template Get(); - - EXPECT_EQ(output_matrix.rows(), expected_num_channels); - EXPECT_EQ(output_matrix.cols(), expected_num_samples_per_packet); - } - } - - void SetupGraphAndHeader() { - this->InitializeGraph(); - this->FillInputHeader(); - } - - // Argument is the expected number of dimensions (channels, columns) in - // the output data from the Calculator under test, which the test should - // know. - void SetupRandomInputPackets() { - constexpr int kNumPackets = 5; - num_samples_per_packet_ = GenerateRandomNonnegInputStream(kNumPackets); - } - - absl::Status Run() { return this->RunGraph(); } - - void CheckResults(int expected_num_channels) { - const auto& output_header = - this->output().header.template Get(); - EXPECT_EQ(this->input_sample_rate_, output_header.sample_rate()); - CheckOutputPacketMetadata(expected_num_channels, num_samples_per_packet_); - - // Sanity check that output packets have non-zero energy. - for (const auto& packet : this->output().packets) { - const Matrix& data = packet.template Get(); - EXPECT_GT(data.squaredNorm(), 0); - } - } - - // Allows SetupRandomInputPackets() to inform CheckResults() about how - // big the packets are supposed to be. - int num_samples_per_packet_; -}; - -constexpr char kMfccCalculator[] = "MfccCalculator"; -typedef FramewiseTransformCalculatorTest - MfccCalculatorTest; -TEST_F(MfccCalculatorTest, AudioSampleRateFromInputHeader) { - audio_sample_rate_ = kAudioSampleRate; - SetupGraphAndHeader(); - SetupRandomInputPackets(); - - MP_EXPECT_OK(Run()); - - CheckResults(options_.mfcc_count()); -} -TEST_F(MfccCalculatorTest, NoAudioSampleRate) { - // Leave audio_sample_rate_ == kUnset, so it is not present in the - // input TimeSeriesHeader; expect failure. - SetupGraphAndHeader(); - SetupRandomInputPackets(); - - EXPECT_FALSE(Run().ok()); -} - -constexpr char kMelSpectrumCalculator[] = "MelSpectrumCalculator"; -typedef FramewiseTransformCalculatorTest - MelSpectrumCalculatorTest; -TEST_F(MelSpectrumCalculatorTest, AudioSampleRateFromInputHeader) { - audio_sample_rate_ = kAudioSampleRate; - SetupGraphAndHeader(); - SetupRandomInputPackets(); - - MP_EXPECT_OK(Run()); - - CheckResults(options_.channel_count()); -} -TEST_F(MelSpectrumCalculatorTest, NoAudioSampleRate) { - // Leave audio_sample_rate_ == kUnset, so it is not present in the - // input TimeSeriesHeader; expect failure. - SetupGraphAndHeader(); - SetupRandomInputPackets(); - - EXPECT_FALSE(Run().ok()); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc deleted file mode 100644 index 1a4210c30..000000000 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2019, 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Defines RationalFactorResampleCalculator. - -#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h" - -#include "audio/dsp/resampler_q.h" - -using audio_dsp::Resampler; - -namespace mediapipe { -absl::Status RationalFactorResampleCalculator::Process(CalculatorContext* cc) { - return ProcessInternal(cc->Inputs().Index(0).Get(), false, cc); -} - -absl::Status RationalFactorResampleCalculator::Close(CalculatorContext* cc) { - if (initial_timestamp_ == Timestamp::Unstarted()) { - return absl::OkStatus(); - } - Matrix empty_input_frame(num_channels_, 0); - return ProcessInternal(empty_input_frame, true, cc); -} - -namespace { -void CopyChannelToVector(const Matrix& matrix, int channel, - std::vector* vec) { - vec->resize(matrix.cols()); - Eigen::Map(vec->data(), vec->size()) = matrix.row(channel); -} - -void CopyVectorToChannel(const std::vector& vec, Matrix* matrix, - int channel) { - if (matrix->cols() == 0) { - matrix->resize(matrix->rows(), vec.size()); - } else { - CHECK_EQ(vec.size(), matrix->cols()); - } - CHECK_LT(channel, matrix->rows()); - matrix->row(channel) = - Eigen::Map(vec.data(), vec.size()); -} -} // namespace - -absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) { - RationalFactorResampleCalculatorOptions resample_options = - cc->Options(); - - if (!resample_options.has_target_sample_rate()) { - return tool::StatusInvalid( - "resample_options doesn't have target_sample_rate."); - } - target_sample_rate_ = resample_options.target_sample_rate(); - - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - - source_sample_rate_ = input_header.sample_rate(); - num_channels_ = input_header.num_channels(); - - // Don't create resamplers for pass-thru (sample rates are equal). - if (source_sample_rate_ != target_sample_rate_) { - resampler_.resize(num_channels_); - for (auto& r : resampler_) { - r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_, - resample_options); - if (!r) { - LOG(ERROR) << "Failed to initialize resampler."; - return absl::UnknownError("Failed to initialize resampler."); - } - } - } - - TimeSeriesHeader* output_header = new TimeSeriesHeader(input_header); - output_header->set_sample_rate(target_sample_rate_); - // The resampler doesn't make guarantees about how many samples will - // be in each packet. - output_header->clear_packet_rate(); - output_header->clear_num_samples(); - - cc->Outputs().Index(0).SetHeader(Adopt(output_header)); - cumulative_output_samples_ = 0; - cumulative_input_samples_ = 0; - initial_timestamp_ = Timestamp::Unstarted(); - check_inconsistent_timestamps_ = - resample_options.check_inconsistent_timestamps(); - return absl::OkStatus(); -} - -absl::Status RationalFactorResampleCalculator::ProcessInternal( - const Matrix& input_frame, bool should_flush, CalculatorContext* cc) { - if (initial_timestamp_ == Timestamp::Unstarted()) { - initial_timestamp_ = cc->InputTimestamp(); - } - - if (check_inconsistent_timestamps_) { - time_series_util::LogWarningIfTimestampIsInconsistent( - cc->InputTimestamp(), initial_timestamp_, cumulative_input_samples_, - source_sample_rate_); - } - Timestamp output_timestamp = - initial_timestamp_ + ((cumulative_output_samples_ / target_sample_rate_) * - Timestamp::kTimestampUnitsPerSecond); - - cumulative_input_samples_ += input_frame.cols(); - std::unique_ptr output_frame(new Matrix(num_channels_, 0)); - if (resampler_.empty()) { - // Sample rates were same for input and output; pass-thru. - *output_frame = input_frame; - } else { - if (!Resample(input_frame, output_frame.get(), should_flush)) { - return absl::UnknownError("Resample() failed."); - } - } - cumulative_output_samples_ += output_frame->cols(); - - if (output_frame->cols() > 0) { - cc->Outputs().Index(0).Add(output_frame.release(), output_timestamp); - } - return absl::OkStatus(); -} - -bool RationalFactorResampleCalculator::Resample(const Matrix& input_frame, - Matrix* output_frame, - bool should_flush) { - std::vector input_vector; - std::vector output_vector; - for (int i = 0; i < input_frame.rows(); ++i) { - CopyChannelToVector(input_frame, i, &input_vector); - if (should_flush) { - resampler_[i]->Flush(&output_vector); - } else { - resampler_[i]->ProcessSamples(input_vector, &output_vector); - } - CopyVectorToChannel(output_vector, output_frame, i); - } - return true; -} - -// static -std::unique_ptr> -RationalFactorResampleCalculator::ResamplerFromOptions( - const double source_sample_rate, const double target_sample_rate, - const RationalFactorResampleCalculatorOptions& options) { - std::unique_ptr> resampler; - const auto& rational_factor_options = - options.resampler_rational_factor_options(); - audio_dsp::QResamplerParams params; - if (rational_factor_options.has_radius() && - rational_factor_options.has_cutoff() && - rational_factor_options.has_kaiser_beta()) { - // Convert RationalFactorResampler kernel parameters to QResampler - // settings. - params.filter_radius_factor = - rational_factor_options.radius() * - std::min(1.0, target_sample_rate / source_sample_rate); - params.cutoff_proportion = 2 * rational_factor_options.cutoff() / - std::min(source_sample_rate, target_sample_rate); - params.kaiser_beta = rational_factor_options.kaiser_beta(); - } - // Set large enough so that the resampling factor between common sample - // rates (e.g. 8kHz, 16kHz, 22.05kHz, 32kHz, 44.1kHz, 48kHz) is exact, and - // that any factor is represented with error less than 0.025%. - params.max_denominator = 2000; - - // NOTE: QResampler supports multichannel resampling, so the code might be - // simplified using a single instance rather than one per channel. - resampler = absl::make_unique>( - source_sample_rate, target_sample_rate, /*num_channels=*/1, params); - if (resampler != nullptr && !resampler->Valid()) { - resampler = std::unique_ptr>(); - } - return resampler; -} - -REGISTER_CALCULATOR(RationalFactorResampleCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.h b/mediapipe/calculators/audio/rational_factor_resample_calculator.h deleted file mode 100644 index 325886dc7..000000000 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.h +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2019, 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_ - -#include -#include -#include - -#include "Eigen/Core" -#include "absl/strings/str_cat.h" -#include "audio/dsp/resampler.h" -#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { -// MediaPipe Calculator for resampling a (vector-valued) -// input time series with a uniform sample rate. The output -// stream's sampling rate is specified by target_sample_rate in the -// RationalFactorResampleCalculatorOptions. The output time series may have -// a varying number of samples per frame. -// -// NOTE: This calculator uses QResampler, despite the name, which supersedes -// RationalFactorResampler. -class RationalFactorResampleCalculator : public CalculatorBase { - public: - struct TestAccess; - - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Single input stream with TimeSeriesHeader. - ); - cc->Outputs().Index(0).Set( - // Resampled stream with TimeSeriesHeader. - ); - return absl::OkStatus(); - } - // Returns FAIL if the input stream header is invalid or if the - // resampler cannot be initialized. - absl::Status Open(CalculatorContext* cc) override; - // Resamples a packet of TimeSeries data. Returns FAIL if the - // resampler state becomes inconsistent. - absl::Status Process(CalculatorContext* cc) override; - // Flushes any remaining state. Returns FAIL if the resampler state - // becomes inconsistent. - absl::Status Close(CalculatorContext* cc) override; - - protected: - typedef audio_dsp::Resampler ResamplerType; - - // Returns a Resampler implementation specified by the - // RationalFactorResampleCalculatorOptions proto. Returns null if the options - // specify an invalid resampler. - static std::unique_ptr ResamplerFromOptions( - const double source_sample_rate, const double target_sample_rate, - const RationalFactorResampleCalculatorOptions& options); - - // Does Timestamp bookkeeping and resampling common to Process() and - // Close(). Returns FAIL if the resampler state becomes - // inconsistent. - absl::Status ProcessInternal(const Matrix& input_frame, bool should_flush, - CalculatorContext* cc); - - // Uses the internal resampler_ objects to actually resample each - // row of the input TimeSeries. Returns false if the resampler - // state becomes inconsistent. - bool Resample(const Matrix& input_frame, Matrix* output_frame, - bool should_flush); - - double source_sample_rate_; - double target_sample_rate_; - int64 cumulative_input_samples_; - int64 cumulative_output_samples_; - Timestamp initial_timestamp_; - bool check_inconsistent_timestamps_; - int num_channels_; - std::vector> resampler_; -}; - -// Test-only access to RationalFactorResampleCalculator methods. -struct RationalFactorResampleCalculator::TestAccess { - static std::unique_ptr ResamplerFromOptions( - const double source_sample_rate, const double target_sample_rate, - const RationalFactorResampleCalculatorOptions& options) { - return RationalFactorResampleCalculator::ResamplerFromOptions( - source_sample_rate, target_sample_rate, options); - } -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_AUDIO_RATIONAL_FACTOR_RESAMPLE_CALCULATOR_H_ diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.proto b/mediapipe/calculators/audio/rational_factor_resample_calculator.proto deleted file mode 100644 index 97d7f202c..000000000 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.proto +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019, 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// NOTE: This calculator uses QResampler, despite the name, which supersedes -// RationalFactorResampler. -message RationalFactorResampleCalculatorOptions { - extend CalculatorOptions { - optional RationalFactorResampleCalculatorOptions ext = 259760074; - } - - // target_sample_rate is the sample rate, in Hertz, of the output - // stream. Required. Must be greater than 0. - optional double target_sample_rate = 1; - - // Parameters for initializing QResampler. See QResampler for more details. - message ResamplerRationalFactorOptions { - // Kernel radius in units of input samples. - optional double radius = 1; - // Anti-aliasing cutoff frequency in Hertz. A reasonable setting is - // 0.45 * min(input_sample_rate, output_sample_rate). - optional double cutoff = 2; - // The Kaiser beta parameter for the kernel window. - optional double kaiser_beta = 3 [default = 6.0]; - } - optional ResamplerRationalFactorOptions resampler_rational_factor_options = 2; - - // Set to false to disable checks for jitter in timestamp values. Useful with - // live audio input. - optional bool check_inconsistent_timestamps = 3 [default = true]; -} diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc b/mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc deleted file mode 100644 index 6ae360303..000000000 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator_test.cc +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h" - -#include - -#include -#include -#include - -#include "Eigen/Core" -#include "audio/dsp/signal_vector_util.h" -#include "mediapipe/calculators/audio/rational_factor_resample_calculator.pb.h" -#include "mediapipe/framework//tool/validate_type.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { -namespace { - -const int kInitialTimestampOffsetMilliseconds = 4; - -class RationalFactorResampleCalculatorTest - : public TimeSeriesCalculatorTest { - protected: - void SetUp() override { - calculator_name_ = "RationalFactorResampleCalculator"; - input_sample_rate_ = 4000.0; - num_input_channels_ = 3; - } - - // Expects two vectors whose lengths are almost the same and whose - // elements are equal (for indices that are present in both). - // - // This is useful because the resampler doesn't make precise - // guarantees about its output size. - void ExpectVectorMostlyFloatEq(const std::vector& expected, - const std::vector& actual) { - // Lengths should be close, but don't have to be equal. - ASSERT_NEAR(expected.size(), actual.size(), 1); - for (int i = 0; i < std::min(expected.size(), actual.size()); ++i) { - EXPECT_FLOAT_EQ(expected[i], actual[i]) << " where i=" << i << "."; - } - } - - // Returns a float value with the sample, channel, and timestamp - // separated by a few orders of magnitude, for easy parsing by - // humans. - double TestValue(int sample, int channel, int timestamp_in_microseconds) { - return timestamp_in_microseconds * 100.0 + sample + channel / 10.0; - } - - // Caller takes ownership of the returned value. - Matrix* NewTestFrame(int num_channels, int num_samples, int timestamp) { - auto matrix = new Matrix(num_channels, num_samples); - for (int c = 0; c < num_channels; ++c) { - for (int i = 0; i < num_samples; ++i) { - (*matrix)(c, i) = TestValue(i, c, timestamp); - } - } - return matrix; - } - - // Initializes and runs the test graph. - absl::Status Run(double output_sample_rate) { - options_.set_target_sample_rate(output_sample_rate); - InitializeGraph(); - - FillInputHeader(); - concatenated_input_samples_.resize(num_input_channels_, 0); - num_input_samples_ = 0; - for (int i = 0; i < 5; ++i) { - int packet_size = (i + 1) * 10; - int timestamp = kInitialTimestampOffsetMilliseconds + - num_input_samples_ / input_sample_rate_ * - Timestamp::kTimestampUnitsPerSecond; - Matrix* data_frame = - NewTestFrame(num_input_channels_, packet_size, timestamp); - - // Keep a reference copy of the input. - // - // conservativeResize() is needed here to preserve the existing - // data. Eigen's resize() resizes without preserving data. - concatenated_input_samples_.conservativeResize( - num_input_channels_, num_input_samples_ + packet_size); - concatenated_input_samples_.rightCols(packet_size) = *data_frame; - num_input_samples_ += packet_size; - - AppendInputPacket(data_frame, timestamp); - } - - return RunGraph(); - } - - void CheckOutputLength(double output_sample_rate) { - double factor = output_sample_rate / input_sample_rate_; - - int num_output_samples = 0; - for (const Packet& packet : output().packets) { - num_output_samples += packet.Get().cols(); - } - - // The exact number of expected samples may vary based on the implementation - // of the resampler since the exact value is not an integer. - const double expected_num_output_samples = num_input_samples_ * factor; - EXPECT_LE(ceil(expected_num_output_samples), num_output_samples); - EXPECT_GE(ceil(expected_num_output_samples) + 11, num_output_samples); - } - - // Checks that output timestamps are consistent with the - // output_sample_rate and output packet sizes. - void CheckOutputPacketTimestamps(double output_sample_rate) { - int num_output_samples = 0; - for (const Packet& packet : output().packets) { - const int expected_timestamp = kInitialTimestampOffsetMilliseconds + - num_output_samples / output_sample_rate * - Timestamp::kTimestampUnitsPerSecond; - EXPECT_NEAR(expected_timestamp, packet.Timestamp().Value(), 1); - num_output_samples += packet.Get().cols(); - } - } - - // Checks that output values from the calculator (which resamples - // packet-by-packet) are consistent with resampling the entire - // signal at once. - void CheckOutputValues(double output_sample_rate) { - for (int i = 0; i < num_input_channels_; ++i) { - auto verification_resampler = - RationalFactorResampleCalculator::TestAccess::ResamplerFromOptions( - input_sample_rate_, output_sample_rate, options_); - - std::vector input_data; - for (int j = 0; j < num_input_samples_; ++j) { - input_data.push_back(concatenated_input_samples_(i, j)); - } - std::vector expected_resampled_data; - std::vector temp; - verification_resampler->ProcessSamples(input_data, &temp); - audio_dsp::VectorAppend(&expected_resampled_data, temp); - verification_resampler->Flush(&temp); - audio_dsp::VectorAppend(&expected_resampled_data, temp); - std::vector actual_resampled_data; - for (const Packet& packet : output().packets) { - Matrix output_frame_row = packet.Get().row(i); - actual_resampled_data.insert( - actual_resampled_data.end(), &output_frame_row(0), - &output_frame_row(0) + output_frame_row.cols()); - } - - ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data); - } - } - - void CheckOutputHeaders(double output_sample_rate) { - const TimeSeriesHeader& output_header = - output().header.Get(); - TimeSeriesHeader expected_header; - expected_header.set_sample_rate(output_sample_rate); - expected_header.set_num_channels(num_input_channels_); - EXPECT_THAT(output_header, mediapipe::EqualsProto(expected_header)); - } - - void CheckOutput(double output_sample_rate) { - CheckOutputLength(output_sample_rate); - CheckOutputPacketTimestamps(output_sample_rate); - CheckOutputValues(output_sample_rate); - CheckOutputHeaders(output_sample_rate); - } - - void CheckOutputUnchanged() { - for (int i = 0; i < num_input_channels_; ++i) { - std::vector expected_resampled_data; - for (int j = 0; j < num_input_samples_; ++j) { - expected_resampled_data.push_back(concatenated_input_samples_(i, j)); - } - std::vector actual_resampled_data; - for (const Packet& packet : output().packets) { - Matrix output_frame_row = packet.Get().row(i); - actual_resampled_data.insert( - actual_resampled_data.end(), &output_frame_row(0), - &output_frame_row(0) + output_frame_row.cols()); - } - ExpectVectorMostlyFloatEq(expected_resampled_data, actual_resampled_data); - } - } - - int num_input_samples_; - Matrix concatenated_input_samples_; -}; - -TEST_F(RationalFactorResampleCalculatorTest, Upsample) { - const double kUpsampleRate = input_sample_rate_ * 1.9; - MP_ASSERT_OK(Run(kUpsampleRate)); - CheckOutput(kUpsampleRate); -} - -TEST_F(RationalFactorResampleCalculatorTest, Downsample) { - const double kDownsampleRate = input_sample_rate_ / 1.9; - MP_ASSERT_OK(Run(kDownsampleRate)); - CheckOutput(kDownsampleRate); -} - -TEST_F(RationalFactorResampleCalculatorTest, UsesRationalFactorResampler) { - const double kUpsampleRate = input_sample_rate_ * 2; - MP_ASSERT_OK(Run(kUpsampleRate)); - CheckOutput(kUpsampleRate); -} - -TEST_F(RationalFactorResampleCalculatorTest, PassthroughIfSampleRateUnchanged) { - const double kUpsampleRate = input_sample_rate_; - MP_ASSERT_OK(Run(kUpsampleRate)); - CheckOutputUnchanged(); -} - -TEST_F(RationalFactorResampleCalculatorTest, FailsOnBadTargetRate) { - ASSERT_FALSE(Run(-999.9).ok()); // Invalid output sample rate. -} - -TEST_F(RationalFactorResampleCalculatorTest, DoesNotDieOnEmptyInput) { - options_.set_target_sample_rate(input_sample_rate_); - InitializeGraph(); - FillInputHeader(); - MP_ASSERT_OK(RunGraph()); - EXPECT_TRUE(output().packets.empty()); -} - -} // anonymous namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/spectrogram_calculator.cc b/mediapipe/calculators/audio/spectrogram_calculator.cc deleted file mode 100644 index bd2234f86..000000000 --- a/mediapipe/calculators/audio/spectrogram_calculator.cc +++ /dev/null @@ -1,452 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Defines SpectrogramCalculator. -#include - -#include -#include -#include -#include - -#include "Eigen/Core" -#include "absl/strings/string_view.h" -#include "audio/dsp/spectrogram/spectrogram.h" -#include "audio/dsp/window_functions.h" -#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/core_proto_inc.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/source_location.h" -#include "mediapipe/framework/port/status_builder.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { - -// MediaPipe Calculator for computing the "spectrogram" (short-time Fourier -// transform squared-magnitude, by default) of a multichannel input -// time series, including optionally overlapping frames. Options are -// specified in SpectrogramCalculatorOptions proto (where names are chosen -// to mirror TimeSeriesFramerCalculator): -// -// Result is a MatrixData record (for single channel input and when the -// allow_multichannel_input flag is false), or a vector of MatrixData records, -// one for each channel (when the allow_multichannel_input flag is set). The -// rows of each spectrogram matrix correspond to the n_fft/2+1 unique complex -// values, or squared/linear/dB magnitudes, depending on the output_type option. -// Each input packet will result in zero or one output packets, each containing -// one Matrix for each channel of the input, where each Matrix has one or more -// columns of spectral values, one for each complete frame of input samples. If -// the input packet contains too few samples to trigger a new output frame, no -// output packet is generated (since zero-length packets are not legal since -// they would result in timestamps that were equal, not strictly increasing). -// -// Output packet Timestamps are set to the beginning of each frame. This is to -// allow calculators downstream from SpectrogramCalculator to have aligned -// Timestamps regardless of a packet's signal length. -// -// Both frame_duration_seconds and frame_overlap_seconds will be -// rounded to the nearest integer number of samples. Conseqently, all output -// frames will be based on the same number of input samples, and each -// analysis frame will advance from its predecessor by the same time step. -class SpectrogramCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Input stream with TimeSeriesHeader. - ); - - SpectrogramCalculatorOptions spectrogram_options = - cc->Options(); - if (!spectrogram_options.allow_multichannel_input()) { - if (spectrogram_options.output_type() == - SpectrogramCalculatorOptions::COMPLEX) { - cc->Outputs().Index(0).Set( - // Complex spectrogram frames with TimeSeriesHeader. - ); - } else { - cc->Outputs().Index(0).Set( - // Spectrogram frames with TimeSeriesHeader. - ); - } - } else { - if (spectrogram_options.output_type() == - SpectrogramCalculatorOptions::COMPLEX) { - cc->Outputs().Index(0).Set>( - // Complex spectrogram frames with MultiStreamTimeSeriesHeader. - ); - } else { - cc->Outputs().Index(0).Set>( - // Spectrogram frames with MultiStreamTimeSeriesHeader. - ); - } - } - return absl::OkStatus(); - } - - // Returns FAIL if the input stream header is invalid. - absl::Status Open(CalculatorContext* cc) override; - - // Outputs at most one packet consisting of a single Matrix with one or - // more columns containing the spectral values from as many input frames - // as are completed by the input samples. Always returns OK. - absl::Status Process(CalculatorContext* cc) override; - - // Performs zero-padding and processing of any remaining samples - // if pad_final_packet is set. - // Returns OK. - absl::Status Close(CalculatorContext* cc) override; - - private: - Timestamp CurrentOutputTimestamp(CalculatorContext* cc) { - if (use_local_timestamp_) { - const Timestamp now = cc->InputTimestamp(); - if (now == Timestamp::Done()) { - // During Close the timestamp is not available, send an estimate. - return last_local_output_timestamp_ + - round(last_completed_frames_ * frame_step_samples() * - Timestamp::kTimestampUnitsPerSecond / input_sample_rate_); - } - last_local_output_timestamp_ = now; - return now; - } - return CumulativeOutputTimestamp(); - } - - Timestamp CumulativeOutputTimestamp() { - // Cumulative output timestamp is the *center* of the next frame to be - // emitted, hence delayed by half a window duration compared to relevant - // input timestamp. - return initial_input_timestamp_ + - round(cumulative_completed_frames_ * frame_step_samples() * - Timestamp::kTimestampUnitsPerSecond / input_sample_rate_); - } - - int frame_step_samples() const { - return frame_duration_samples_ - frame_overlap_samples_; - } - - // Take the next set of input samples, already translated into a - // vector and pass them to the spectrogram object. - // Convert the output of the spectrogram object into a Matrix (or an - // Eigen::MatrixXcf if complex-valued output is requested) and pass to - // MediaPipe output. - absl::Status ProcessVector(const Matrix& input_stream, CalculatorContext* cc); - - // Templated function to process either real- or complex-output spectrogram. - template - absl::Status ProcessVectorToOutput( - const Matrix& input_stream, - const OutputMatrixType postprocess_output_fn(const OutputMatrixType&), - CalculatorContext* cc); - - // Use the MediaPipe timestamp instead of the estimated one. Useful when the - // data is intermittent. - bool use_local_timestamp_; - Timestamp last_local_output_timestamp_; - - double input_sample_rate_; - bool pad_final_packet_; - int frame_duration_samples_; - int frame_overlap_samples_; - // How many samples we've been passed, used for checking input time stamps. - int64 cumulative_input_samples_; - // How many frames we've emitted, used for calculating output time stamps. - int64 cumulative_completed_frames_; - // How many frames were emitted last, used for estimating the timestamp on - // Close when use_local_timestamp_ is true; - int64 last_completed_frames_; - Timestamp initial_input_timestamp_; - int num_input_channels_; - // How many frequency bins we emit (=N_FFT/2 + 1). - int num_output_channels_; - // Which output type? - int output_type_; - // Output type: mono or multichannel. - bool allow_multichannel_input_; - // Vector of Spectrogram objects, one for each channel. - std::vector> spectrogram_generators_; - // Fixed scale factor applied to output values (regardless of type). - double output_scale_; - - static const float kLnPowerToDb; -}; -REGISTER_CALCULATOR(SpectrogramCalculator); - -// Factor to convert ln(magnitude_squared) to deciBels = 10.0/ln(10.0). -const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518; - -absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) { - SpectrogramCalculatorOptions spectrogram_options = - cc->Options(); - - use_local_timestamp_ = spectrogram_options.use_local_timestamp(); - - if (spectrogram_options.frame_duration_seconds() <= 0.0) { - // TODO: return an error. - } - if (spectrogram_options.frame_overlap_seconds() >= - spectrogram_options.frame_duration_seconds()) { - // TODO: return an error. - } - if (spectrogram_options.frame_overlap_seconds() < 0.0) { - // TODO: return an error. - } - - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - - input_sample_rate_ = input_header.sample_rate(); - num_input_channels_ = input_header.num_channels(); - - if (!spectrogram_options.allow_multichannel_input() && - num_input_channels_ != 1) { - // TODO: return an error. - } - - frame_duration_samples_ = - round(spectrogram_options.frame_duration_seconds() * input_sample_rate_); - frame_overlap_samples_ = - round(spectrogram_options.frame_overlap_seconds() * input_sample_rate_); - - pad_final_packet_ = spectrogram_options.pad_final_packet(); - output_type_ = spectrogram_options.output_type(); - allow_multichannel_input_ = spectrogram_options.allow_multichannel_input(); - - output_scale_ = spectrogram_options.output_scale(); - - std::vector window; - switch (spectrogram_options.window_type()) { - case SpectrogramCalculatorOptions::COSINE: - audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - case SpectrogramCalculatorOptions::HANN: - audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - case SpectrogramCalculatorOptions::HAMMING: - audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - } - - // Propagate settings down to the actual Spectrogram object. - spectrogram_generators_.clear(); - for (int i = 0; i < num_input_channels_; i++) { - spectrogram_generators_.push_back( - std::unique_ptr(new audio_dsp::Spectrogram())); - spectrogram_generators_[i]->Initialize(window, frame_step_samples()); - } - - num_output_channels_ = - spectrogram_generators_[0]->output_frequency_channels(); - std::unique_ptr output_header( - new TimeSeriesHeader(input_header)); - // Store the actual sample rate of the input audio in the TimeSeriesHeader - // so that subsequent calculators can figure out the frequency scale of - // our output. - output_header->set_audio_sample_rate(input_sample_rate_); - // Setup rest of output header. - output_header->set_num_channels(num_output_channels_); - output_header->set_sample_rate(input_sample_rate_ / frame_step_samples()); - // Although we usually generate one output packet for each input - // packet, this might not be true for input packets whose size is smaller - // than the analysis window length. So we clear output_header.packet_rate - // because we can't guarantee a constant packet rate. Similarly, the number - // of output frames per packet depends on the input packet, so we also clear - // output_header.num_samples. - output_header->clear_packet_rate(); - output_header->clear_num_samples(); - if (!spectrogram_options.allow_multichannel_input()) { - cc->Outputs().Index(0).SetHeader(Adopt(output_header.release())); - } else { - std::unique_ptr multichannel_output_header( - new MultiStreamTimeSeriesHeader()); - *multichannel_output_header->mutable_time_series_header() = *output_header; - multichannel_output_header->set_num_streams(num_input_channels_); - cc->Outputs().Index(0).SetHeader( - Adopt(multichannel_output_header.release())); - } - cumulative_completed_frames_ = 0; - last_completed_frames_ = 0; - initial_input_timestamp_ = Timestamp::Unstarted(); - if (use_local_timestamp_) { - // Inform the framework that the calculator will output packets at the same - // timestamps as input packets to enable packet queueing optimizations. The - // final packet (emitted from Close()) does not follow this rule but it's - // sufficient that its timestamp is strictly greater than the timestamp of - // the previous packet. - cc->SetOffset(0); - } - return absl::OkStatus(); -} - -absl::Status SpectrogramCalculator::Process(CalculatorContext* cc) { - if (initial_input_timestamp_ == Timestamp::Unstarted()) { - initial_input_timestamp_ = cc->InputTimestamp(); - } - - const Matrix& input_stream = cc->Inputs().Index(0).Get(); - if (input_stream.rows() != num_input_channels_) { - // TODO: return an error. - } - - cumulative_input_samples_ += input_stream.cols(); - - return ProcessVector(input_stream, cc); -} - -template -absl::Status SpectrogramCalculator::ProcessVectorToOutput( - const Matrix& input_stream, - const OutputMatrixType postprocess_output_fn(const OutputMatrixType&), - CalculatorContext* cc) { - std::unique_ptr> spectrogram_matrices( - new std::vector()); - std::vector> output_vectors; - - // Compute a spectrogram for each channel. - int num_output_time_frames; - for (int channel = 0; channel < input_stream.rows(); ++channel) { - output_vectors.clear(); - - // Copy one row (channel) of the input matrix into the std::vector. - std::vector input_vector(input_stream.cols()); - Eigen::Map(&input_vector[0], 1, input_vector.size()) = - input_stream.row(channel); - - if (!spectrogram_generators_[channel]->ComputeSpectrogram( - input_vector, &output_vectors)) { - return absl::Status(absl::StatusCode::kInternal, - "Spectrogram returned failure"); - } - if (channel == 0) { - // Record the number of time frames we expect from each channel. - num_output_time_frames = output_vectors.size(); - } else { - RET_CHECK_EQ(output_vectors.size(), num_output_time_frames) - << "Inconsistent spectrogram time frames for channel " << channel; - } - // Skip remaining processing if there are too few input samples to trigger - // any output frames. - if (!output_vectors.empty()) { - // Translate the returned values into a matrix of output frames. - OutputMatrixType output_frames(num_output_channels_, - output_vectors.size()); - for (int frame = 0; frame < output_vectors.size(); ++frame) { - Eigen::Map frame_map( - &output_vectors[frame][0], output_vectors[frame].size(), 1); - // The underlying dsp object returns squared magnitudes; here - // we optionally translate to linear magnitude or dB. - output_frames.col(frame) = - output_scale_ * postprocess_output_fn(frame_map); - } - spectrogram_matrices->push_back(output_frames); - } - } - // If the input is very short, there may not be enough accumulated, - // unprocessed samples to cause any new frames to be generated by - // the spectrogram object. If so, we don't want to emit - // a packet at all. - if (!spectrogram_matrices->empty()) { - RET_CHECK_EQ(spectrogram_matrices->size(), input_stream.rows()) - << "Inconsistent number of spectrogram channels."; - if (allow_multichannel_input_) { - cc->Outputs().Index(0).Add(spectrogram_matrices.release(), - CurrentOutputTimestamp(cc)); - } else { - cc->Outputs().Index(0).Add( - new OutputMatrixType(spectrogram_matrices->at(0)), - CurrentOutputTimestamp(cc)); - } - cumulative_completed_frames_ += output_vectors.size(); - last_completed_frames_ = output_vectors.size(); - if (!use_local_timestamp_) { - // In non-local timestamp mode the timestamp of the next packet will be - // equal to CumulativeOutputTimestamp(). Inform the framework about this - // fact to enable packet queueing optimizations. - cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp()); - } - } - return absl::OkStatus(); -} - -absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream, - CalculatorContext* cc) { - switch (output_type_) { - // These blocks deliberately ignore clang-format to preserve the - // "silhouette" of the different cases. - // clang-format off - case SpectrogramCalculatorOptions::COMPLEX: { - return ProcessVectorToOutput( - input_stream, - +[](const Eigen::MatrixXcf& col) -> const Eigen::MatrixXcf { - return col; - }, cc); - } - case SpectrogramCalculatorOptions::SQUARED_MAGNITUDE: { - return ProcessVectorToOutput( - input_stream, - +[](const Matrix& col) -> const Matrix { - return col; - }, cc); - } - case SpectrogramCalculatorOptions::LINEAR_MAGNITUDE: { - return ProcessVectorToOutput( - input_stream, - +[](const Matrix& col) -> const Matrix { - return col.array().sqrt().matrix(); - }, cc); - } - case SpectrogramCalculatorOptions::DECIBELS: { - return ProcessVectorToOutput( - input_stream, - +[](const Matrix& col) -> const Matrix { - return kLnPowerToDb * col.array().log().matrix(); - }, cc); - } - // clang-format on - default: { - return absl::Status(absl::StatusCode::kInvalidArgument, - "Unrecognized spectrogram output type."); - } - } -} - -absl::Status SpectrogramCalculator::Close(CalculatorContext* cc) { - if (cumulative_input_samples_ > 0 && pad_final_packet_) { - // We can flush any remaining samples by sending frame_step_samples - 1 - // zeros to the Process method, and letting it do its thing, - // UNLESS we have fewer than one window's worth of samples, in which case - // we pad to exactly one frame_duration_samples. - // Release the memory for the Spectrogram objects. - int required_padding_samples = frame_step_samples() - 1; - if (cumulative_input_samples_ < frame_duration_samples_) { - required_padding_samples = - frame_duration_samples_ - cumulative_input_samples_; - } - return ProcessVector( - Matrix::Zero(num_input_channels_, required_padding_samples), cc); - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/spectrogram_calculator.proto b/mediapipe/calculators/audio/spectrogram_calculator.proto deleted file mode 100644 index b721117d4..000000000 --- a/mediapipe/calculators/audio/spectrogram_calculator.proto +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message SpectrogramCalculatorOptions { - extend CalculatorOptions { - optional SpectrogramCalculatorOptions ext = 76186688; - } - - // Options mirror those of TimeSeriesFramerCalculator. - - // Analysis window duration in seconds. Required. Must be greater than 0. - // (Note: the spectrogram DFT length will be the smallest power-of-2 - // sample count that can hold this duration.) - optional double frame_duration_seconds = 1; - - // Duration of overlap between adjacent windows. - // Hence, frame_rate = 1/(frame_duration_seconds - frame_overlap_seconds). - // Required that 0 <= frame_overlap_seconds < frame_duration_seconds. - optional double frame_overlap_seconds = 2 [default = 0.0]; - - // Whether to pad the final packet with zeros. If true, guarantees that - // all input samples will output. If set to false, any partial packet - // at the end of the stream will be dropped. - optional bool pad_final_packet = 3 [default = true]; - - // Output value type can be squared-magnitude, linear-magnitude, - // deciBels (dB, = 20*log10(linear_magnitude)), or std::complex. - enum OutputType { - SQUARED_MAGNITUDE = 0; - LINEAR_MAGNITUDE = 1; - DECIBELS = 2; - COMPLEX = 3; - } - optional OutputType output_type = 4 [default = SQUARED_MAGNITUDE]; - - // If set to true then the output will be a vector of spectrograms, one for - // each channel and the stream will have a MultiStreamTimeSeriesHeader. - optional bool allow_multichannel_input = 5 [default = false]; - - // Which window to use when computing the FFT. - enum WindowType { - HANN = 0; - HAMMING = 1; - COSINE = 2; - } - optional WindowType window_type = 6 [default = HANN]; - - // Support a fixed multiplicative scaling of the output. This is applied - // uniformly regardless of output type (i.e., even dBs are multiplied, not - // offset). - optional double output_scale = 7 [default = 1.0]; - - // If use_local_timestamp is true, the output packet's timestamp is based on - // the last sample of the packet and it's inferred from the latest input - // packet's timestamp. If false, the output packet's timestamp is based on - // the cumulative timestamping, which is inferred from the intial input - // timestamp and the cumulative number of samples. - optional bool use_local_timestamp = 8 [default = false]; -} diff --git a/mediapipe/calculators/audio/spectrogram_calculator_test.cc b/mediapipe/calculators/audio/spectrogram_calculator_test.cc deleted file mode 100644 index 3c2b8435d..000000000 --- a/mediapipe/calculators/audio/spectrogram_calculator_test.cc +++ /dev/null @@ -1,895 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include -#include -#include - -#include "Eigen/Core" -#include "audio/dsp/number_util.h" -#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/benchmark.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { -namespace { - -const int kInitialTimestampOffsetMicroseconds = 4; - -class SpectrogramCalculatorTest - : public TimeSeriesCalculatorTest { - protected: - void SetUp() override { - calculator_name_ = "SpectrogramCalculator"; - input_sample_rate_ = 4000.0; - num_input_channels_ = 1; - } - - // Initializes and runs the test graph. - absl::Status Run() { - // Now that options are set, we can set up some internal constants. - frame_duration_samples_ = - round(options_.frame_duration_seconds() * input_sample_rate_); - frame_step_samples_ = - frame_duration_samples_ - - round(options_.frame_overlap_seconds() * input_sample_rate_); - // The magnitude of the 0th FFT bin (DC) should be sum(input.*window); - // for an input identically 1.0, this is just sum(window). The average - // value of our Hann window is 0.5, hence this is the expected squared- - // magnitude output value in the DC bin for constant input of 1.0. - expected_dc_squared_magnitude_ = - pow((static_cast(frame_duration_samples_) * 0.5), 2.0); - - return RunGraph(); - } - - // Creates test multichannel input with specified packet sizes and containing - // a constant-frequency sinusoid that maintains phase between adjacent - // packets. - void SetupCosineInputPackets(const std::vector& packet_sizes_samples, - float cosine_frequency_hz) { - int total_num_input_samples = 0; - for (int packet_size_samples : packet_sizes_samples) { - double packet_start_time_seconds = - kInitialTimestampOffsetMicroseconds * 1e-6 + - total_num_input_samples / input_sample_rate_; - double packet_end_time_seconds = - packet_start_time_seconds + packet_size_samples / input_sample_rate_; - double angular_freq = 2 * M_PI * cosine_frequency_hz; - Matrix* packet_data = - new Matrix(num_input_channels_, packet_size_samples); - // Use Eigen's vectorized cos() function to fill the vector with a - // sinusoid of appropriate frequency & phase. - for (int i = 0; i < num_input_channels_; i++) { - packet_data->row(i) = - Eigen::ArrayXf::LinSpaced(packet_size_samples, - packet_start_time_seconds * angular_freq, - packet_end_time_seconds * angular_freq) - .cos() - .transpose(); - } - int64 input_timestamp = round(packet_start_time_seconds * - Timestamp::kTimestampUnitsPerSecond); - AppendInputPacket(packet_data, input_timestamp); - total_num_input_samples += packet_size_samples; - } - } - - // Setup a sequence of input packets of specified sizes, each filled - // with samples of 1.0. - void SetupConstantInputPackets(const std::vector& packet_sizes_samples) { - // A 0 Hz cosine is identically 1.0 for all samples. - SetupCosineInputPackets(packet_sizes_samples, 0.0); - } - - // Setup a sequence of input packets of specified sizes, each containing a - // single sample of 1.0 at a specified offset. - void SetupImpulseInputPackets( - const std::vector& packet_sizes_samples, - const std::vector& impulse_offsets_samples) { - int total_num_input_samples = 0; - for (int i = 0; i < packet_sizes_samples.size(); ++i) { - double packet_start_time_seconds = - kInitialTimestampOffsetMicroseconds * 1e-6 + - total_num_input_samples / input_sample_rate_; - int64 input_timestamp = round(packet_start_time_seconds * - Timestamp::kTimestampUnitsPerSecond); - std::unique_ptr impulse( - new Matrix(Matrix::Zero(1, packet_sizes_samples[i]))); - (*impulse)(0, impulse_offsets_samples[i]) = 1.0; - AppendInputPacket(impulse.release(), input_timestamp); - total_num_input_samples += packet_sizes_samples[i]; - } - } - - // Creates test multichannel input with specified packet sizes and containing - // constant input packets for the even channels and constant-frequency - // sinusoid that maintains phase between adjacent packets for the odd - // channels. - void SetupMultichannelInputPackets( - const std::vector& packet_sizes_samples, float cosine_frequency_hz) { - int total_num_input_samples = 0; - for (int packet_size_samples : packet_sizes_samples) { - double packet_start_time_seconds = - kInitialTimestampOffsetMicroseconds * 1e-6 + - total_num_input_samples / input_sample_rate_; - double packet_end_time_seconds = - packet_start_time_seconds + packet_size_samples / input_sample_rate_; - double angular_freq; - Matrix* packet_data = - new Matrix(num_input_channels_, packet_size_samples); - // Use Eigen's vectorized cos() function to fill the vector with a - // sinusoid of appropriate frequency & phase. - for (int i = 0; i < num_input_channels_; i++) { - if (i % 2 == 0) { - angular_freq = 0; - } else { - angular_freq = 2 * M_PI * cosine_frequency_hz; - } - packet_data->row(i) = - Eigen::ArrayXf::LinSpaced(packet_size_samples, - packet_start_time_seconds * angular_freq, - packet_end_time_seconds * angular_freq) - .cos() - .transpose(); - } - int64 input_timestamp = round(packet_start_time_seconds * - Timestamp::kTimestampUnitsPerSecond); - AppendInputPacket(packet_data, input_timestamp); - total_num_input_samples += packet_size_samples; - } - } - - // Return vector of the numbers of frames in each output packet. - std::vector OutputFramesPerPacket() { - std::vector frame_counts; - for (const Packet& packet : output().packets) { - const Matrix& matrix = packet.Get(); - frame_counts.push_back(matrix.cols()); - } - return frame_counts; - } - - // Checks output headers and Timestamps. - void CheckOutputHeadersAndTimestamps() { - const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_); - - TimeSeriesHeader expected_header = input().header.Get(); - expected_header.set_num_channels(fft_size / 2 + 1); - // The output header sample rate should depend on the output frame step. - expected_header.set_sample_rate(input_sample_rate_ / frame_step_samples_); - // SpectrogramCalculator stores the sample rate of the input in - // the TimeSeriesHeader. - expected_header.set_audio_sample_rate(input_sample_rate_); - // We expect the output header to have num_samples and packet_rate unset. - expected_header.clear_num_samples(); - expected_header.clear_packet_rate(); - if (!options_.allow_multichannel_input()) { - ExpectOutputHeaderEquals(expected_header); - } else { - EXPECT_THAT(output() - .header.template Get() - .time_series_header(), - mediapipe::EqualsProto(expected_header)); - EXPECT_THAT(output() - .header.template Get() - .num_streams(), - num_input_channels_); - } - - int cumulative_output_frames = 0; - // The timestamps coming out of the spectrogram correspond to the - // middle of the first frame's window, hence frame_duration_samples_/2 - // term. We use frame_duration_samples_ because that is how it is - // actually quantized inside spectrogram. - const double packet_timestamp_offset_seconds = - kInitialTimestampOffsetMicroseconds * 1e-6; - const double frame_step_seconds = frame_step_samples_ / input_sample_rate_; - - Timestamp initial_timestamp = Timestamp::Unstarted(); - - for (const Packet& packet : output().packets) { - // This is the timestamp we expect based on how the spectrogram should - // behave (advancing by one step's worth of input samples each frame). - const double expected_timestamp_seconds = - packet_timestamp_offset_seconds + - cumulative_output_frames * frame_step_seconds; - const int64 expected_timestamp_ticks = - expected_timestamp_seconds * Timestamp::kTimestampUnitsPerSecond; - EXPECT_EQ(expected_timestamp_ticks, packet.Timestamp().Value()); - // Accept the timestamp of the first packet as the baseline for checking - // the remainder. - if (initial_timestamp == Timestamp::Unstarted()) { - initial_timestamp = packet.Timestamp(); - } - // Also check that the timestamp is consistent with the sample_rate - // in the output stream's TimeSeriesHeader. - EXPECT_TRUE(time_series_util::LogWarningIfTimestampIsInconsistent( - packet.Timestamp(), initial_timestamp, cumulative_output_frames, - expected_header.sample_rate())); - if (!options_.allow_multichannel_input()) { - if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) { - const Eigen::MatrixXcf& matrix = packet.Get(); - cumulative_output_frames += matrix.cols(); - } else { - const Matrix& matrix = packet.Get(); - cumulative_output_frames += matrix.cols(); - } - } else { - if (options_.output_type() == SpectrogramCalculatorOptions::COMPLEX) { - const Eigen::MatrixXcf& matrix = - packet.Get>().at(0); - cumulative_output_frames += matrix.cols(); - } else { - const Matrix& matrix = packet.Get>().at(0); - cumulative_output_frames += matrix.cols(); - } - } - } - } - - // Verify that the bin corresponding to the specified frequency - // is the largest one in one particular frame of a single packet. - void CheckPeakFrequencyInPacketFrame(const Packet& packet, int frame, - float frequency) { - const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_); - const int target_bin = - round((frequency / input_sample_rate_) * static_cast(fft_size)); - - const Matrix& matrix = packet.Get(); - // Stop here if the requested frame is not in this packet. - ASSERT_GT(matrix.cols(), frame); - - int actual_largest_bin; - matrix.col(frame).maxCoeff(&actual_largest_bin); - EXPECT_EQ(actual_largest_bin, target_bin); - } - - // Verify that the bin corresponding to the specified frequency - // is the largest one in one particular frame of a single spectrogram Matrix. - void CheckPeakFrequencyInMatrix(const Matrix& matrix, int frame, - float frequency) { - const int fft_size = audio_dsp::NextPowerOfTwo(frame_duration_samples_); - const int target_bin = - round((frequency / input_sample_rate_) * static_cast(fft_size)); - - // Stop here if the requested frame is not in this packet. - ASSERT_GT(matrix.cols(), frame); - - int actual_largest_bin; - matrix.col(frame).maxCoeff(&actual_largest_bin); - EXPECT_EQ(actual_largest_bin, target_bin); - } - - int frame_duration_samples_; - int frame_step_samples_; - // Expected DC output for a window of pure 1.0, set when window length - // is set. - float expected_dc_squared_magnitude_; -}; - -TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationNoOverlap) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(0.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {500, 200}; - const std::vector expected_output_packet_sizes = {5, 2}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, IntegerFrameDurationSomeOverlap) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {500, 200}; - // complete_output_frames = 1 + floor((input_samples - window_length)/step) - // = 1 + floor((500 - 100)/40) = 1 + 10 = 11 for the first packet - // = 1 + floor((700 - 100)/40) = 1 + 15 = 16 for the whole stream - // so expect 16 - 11 = 5 in the second packet. - const std::vector expected_output_packet_sizes = {11, 5}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, NonintegerFrameDurationAndOverlap) { - options_.set_frame_duration_seconds(98.5 / input_sample_rate_); - options_.set_frame_overlap_seconds(58.4 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {500, 200}; - // now frame_duration_samples will be 99 (rounded), and frame_step_samples - // will be (99-58) = 41, so the first packet of 500 samples will generate - // 1 + floor(500-99)/41 = 10 samples. - const std::vector expected_output_packet_sizes = {10, 5}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, ShortInitialPacketNoOverlap) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(0.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {90, 100, 110}; - // The first input packet is too small to generate any frames, - // but zero-length packets would result in a timestamp monotonicity - // violation, so they are suppressed. Thus, only the second and third - // input packets generate output packets. - const std::vector expected_output_packet_sizes = {1, 2}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, TrailingSamplesNoPad) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {140, 90}; - const std::vector expected_output_packet_sizes = {2, 2}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, NoTrailingSamplesWithPad) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(true); - const std::vector input_packet_sizes = {140, 80}; - const std::vector expected_output_packet_sizes = {2, 2}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, TrailingSamplesWithPad) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(true); - const std::vector input_packet_sizes = {140, 90}; - // In contrast to NoTrailingSamplesWithPad and TrailingSamplesNoPad, - // this time we get an extra frame in an extra final packet. - const std::vector expected_output_packet_sizes = {2, 2, 1}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, VeryShortInputWillPad) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(true); - const std::vector input_packet_sizes = {30}; - const std::vector expected_output_packet_sizes = {1}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, VeryShortInputZeroOutputFramesIfNoPad) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - const std::vector input_packet_sizes = {90}; - const std::vector expected_output_packet_sizes = {}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, DCSignalIsPeakBin) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - const std::vector input_packet_sizes = {140}; // Gives 2 output frames. - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - const float dc_frequency_hz = 0.0; - CheckPeakFrequencyInPacketFrame(output().packets[0], 0, dc_frequency_hz); - CheckPeakFrequencyInPacketFrame(output().packets[0], 1, dc_frequency_hz); -} - -TEST_F(SpectrogramCalculatorTest, A440ToneIsPeakBin) { - const std::vector input_packet_sizes = { - 460}; // 100 + 9*40 for 10 frames. - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - int num_output_frames = output().packets[0].Get().cols(); - for (int frame = 0; frame < num_output_frames; ++frame) { - CheckPeakFrequencyInPacketFrame(output().packets[0], frame, - tone_frequency_hz); - } -} - -TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRight) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ(output().packets[0].Get()(0, 0), - expected_dc_squared_magnitude_); -} - -TEST_F(SpectrogramCalculatorTest, DefaultOutputIsSquaredMagnitude) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - // Let the output_type be its default - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ(output().packets[0].Get()(0, 0), - expected_dc_squared_magnitude_); -} - -TEST_F(SpectrogramCalculatorTest, LinearMagnitudeOutputLooksRight) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::LINEAR_MAGNITUDE); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ(output().packets[0].Get()(0, 0), - std::sqrt(expected_dc_squared_magnitude_)); -} - -TEST_F(SpectrogramCalculatorTest, DbMagnitudeOutputLooksRight) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ(output().packets[0].Get()(0, 0), - 10.0 * std::log10(expected_dc_squared_magnitude_)); -} - -TEST_F(SpectrogramCalculatorTest, OutputScalingLooksRight) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::DECIBELS); - double output_scale = 2.5; - options_.set_output_scale(output_scale); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ( - output().packets[0].Get()(0, 0), - output_scale * 10.0 * std::log10(expected_dc_squared_magnitude_)); -} - -TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRight) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Setup packets with DC input (non-zero constant value). - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_FLOAT_EQ(std::norm(output().packets[0].Get()(0, 0)), - expected_dc_squared_magnitude_); -} - -TEST_F(SpectrogramCalculatorTest, ComplexOutputLooksRightForImpulses) { - const int frame_size_samples = 100; - options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_); - options_.set_frame_overlap_seconds(0.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX); - const std::vector input_packet_sizes = {frame_size_samples, - frame_size_samples}; - const std::vector input_packet_impulse_offsets = {49, 50}; - - InitializeGraph(); - FillInputHeader(); - - // Make two impulse packets offset one sample from each other - SetupImpulseInputPackets(input_packet_sizes, input_packet_impulse_offsets); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - const int num_buckets = - (audio_dsp::NextPowerOfTwo(frame_size_samples) / 2) + 1; - const float precision = 0.01f; - auto norm_fn = [](const std::complex& cf) { return std::norm(cf); }; - - // Both impulses should have (approximately) constant power across all - // frequency bins - EXPECT_TRUE(output() - .packets[0] - .Get() - .unaryExpr(norm_fn) - .isApproxToConstant(1.0f, precision)); - EXPECT_TRUE(output() - .packets[1] - .Get() - .unaryExpr(norm_fn) - .isApproxToConstant(1.0f, precision)); - - // Because the second Packet's impulse is delayed by exactly one sample with - // respect to the first Packet's impulse, the second impulse should have - // greater phase, and in the highest frequency bin, the real part should - // (approximately) flip sign from the first Packet to the second - EXPECT_LT(std::arg(output().packets[0].Get()(1, 0)), - std::arg(output().packets[1].Get()(1, 0))); - const float highest_bucket_real_ratio = - output().packets[0].Get()(num_buckets - 1, 0).real() / - output().packets[1].Get()(num_buckets - 1, 0).real(); - EXPECT_NEAR(highest_bucket_real_ratio, -1.0f, precision); -} - -TEST_F(SpectrogramCalculatorTest, SquaredMagnitudeOutputLooksRightForNonDC) { - const int frame_size_samples = 100; - options_.set_frame_duration_seconds(frame_size_samples / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_output_type(SpectrogramCalculatorOptions::SQUARED_MAGNITUDE); - const std::vector input_packet_sizes = {140}; - - InitializeGraph(); - FillInputHeader(); - // Make the tone have an integral number of cycles within the window - const int target_bin = 16; - const int fft_size = audio_dsp::NextPowerOfTwo(frame_size_samples); - const float tone_frequency_hz = target_bin * (input_sample_rate_ / fft_size); - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - // For a non-DC bin, the magnitude will be split between positive and - // negative frequency bins, so it should about be half-magnitude - // = quarter-power. - // It's not quite exact because of the interference from the hann(100) - // spread from the negative-frequency half. - EXPECT_GT(output().packets[0].Get()(target_bin, 0), - 0.98 * expected_dc_squared_magnitude_ / 4.0); - EXPECT_LT(output().packets[0].Get()(target_bin, 0), - 1.02 * expected_dc_squared_magnitude_ / 4.0); -} - -TEST_F(SpectrogramCalculatorTest, ZeroOutputsForZeroInputsWithPaddingEnabled) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(true); - const std::vector input_packet_sizes = {}; - const std::vector expected_output_packet_sizes = {}; - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(OutputFramesPerPacket(), expected_output_packet_sizes); -} - -TEST_F(SpectrogramCalculatorTest, NumChannelsIsRight) { - const std::vector input_packet_sizes = {460}; - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - options_.set_allow_multichannel_input(true); - num_input_channels_ = 3; - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - EXPECT_EQ(output().packets[0].Get>().size(), - num_input_channels_); -} - -TEST_F(SpectrogramCalculatorTest, NumSamplesAndPacketRateAreCleared) { - num_input_samples_ = 500; - input_packet_rate_ = 1.0; - const std::vector input_packet_sizes = {num_input_samples_}; - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(0.0); - options_.set_pad_final_packet(false); - - InitializeGraph(); - FillInputHeader(); - SetupConstantInputPackets(input_packet_sizes); - - MP_ASSERT_OK(Run()); - - const TimeSeriesHeader& output_header = - output().header.Get(); - EXPECT_FALSE(output_header.has_num_samples()); - EXPECT_FALSE(output_header.has_packet_rate()); -} - -TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramSizesAreRight) { - const std::vector input_packet_sizes = {420}; // less than 10 frames - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - options_.set_allow_multichannel_input(true); - num_input_channels_ = 10; - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - auto spectrograms = output().packets[0].Get>(); - EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_); - int spectrogram_num_rows = spectrograms[0].rows(); - int spectrogram_num_cols = spectrograms[0].cols(); - for (int i = 1; i < num_input_channels_; i++) { - EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows()); - EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols()); - } -} - -TEST_F(SpectrogramCalculatorTest, MultichannelSpectrogramValuesAreRight) { - const std::vector input_packet_sizes = { - 460}; // 100 + 9*40 for 10 frames. - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_allow_multichannel_input(true); - num_input_channels_ = 10; - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupMultichannelInputPackets(input_packet_sizes, tone_frequency_hz); - - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - auto spectrograms = output().packets[0].Get>(); - int num_output_frames = spectrograms[0].cols(); - for (int i = 0; i < num_input_channels_; i++) { - for (int frame = 0; frame < num_output_frames; ++frame) { - if (i % 2 == 0) { - CheckPeakFrequencyInMatrix(spectrograms[i], frame, 0); - } else { - CheckPeakFrequencyInMatrix(spectrograms[i], frame, tone_frequency_hz); - } - } - } -} - -TEST_F(SpectrogramCalculatorTest, MultichannelHandlesShortInitialPacket) { - // First packet is less than one frame, but second packet should trigger a - // complete frame from all channels. - const std::vector input_packet_sizes = {50, 50}; - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - options_.set_allow_multichannel_input(true); - num_input_channels_ = 2; - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - auto spectrograms = output().packets[0].Get>(); - EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_); - int spectrogram_num_rows = spectrograms[0].rows(); - int spectrogram_num_cols = spectrograms[0].cols(); - for (int i = 1; i < num_input_channels_; i++) { - EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows()); - EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols()); - } -} - -TEST_F(SpectrogramCalculatorTest, - MultichannelComplexHandlesShortInitialPacket) { - // First packet is less than one frame, but second packet should trigger a - // complete frame from all channels, even for complex output. - const std::vector input_packet_sizes = {50, 50}; - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(60.0 / input_sample_rate_); - options_.set_pad_final_packet(false); - options_.set_allow_multichannel_input(true); - options_.set_output_type(SpectrogramCalculatorOptions::COMPLEX); - num_input_channels_ = 2; - InitializeGraph(); - FillInputHeader(); - const float tone_frequency_hz = 440.0; - SetupCosineInputPackets(input_packet_sizes, tone_frequency_hz); - MP_ASSERT_OK(Run()); - - CheckOutputHeadersAndTimestamps(); - auto spectrograms = output().packets[0].Get>(); - EXPECT_FLOAT_EQ(spectrograms.size(), num_input_channels_); - int spectrogram_num_rows = spectrograms[0].rows(); - int spectrogram_num_cols = spectrograms[0].cols(); - for (int i = 1; i < num_input_channels_; i++) { - EXPECT_EQ(spectrogram_num_rows, spectrograms[i].rows()); - EXPECT_EQ(spectrogram_num_cols, spectrograms[i].cols()); - } -} - -void BM_ProcessDC(benchmark::State& state) { - CalculatorGraphConfig::Node node_config; - node_config.set_calculator("SpectrogramCalculator"); - node_config.add_input_stream("input_audio"); - node_config.add_output_stream("output_spectrogram"); - - SpectrogramCalculatorOptions* options = - node_config.mutable_options()->MutableExtension( - SpectrogramCalculatorOptions::ext); - options->set_frame_duration_seconds(0.010); - options->set_frame_overlap_seconds(0.0); - options->set_pad_final_packet(false); - *node_config.mutable_options()->MutableExtension( - SpectrogramCalculatorOptions::ext) = *options; - - int num_input_channels = 1; - int packet_size_samples = 1600000; - TimeSeriesHeader* header = new TimeSeriesHeader(); - header->set_sample_rate(16000.0); - header->set_num_channels(num_input_channels); - - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).header = Adopt(header); - - Matrix* payload = new Matrix( - Matrix::Constant(num_input_channels, packet_size_samples, 1.0)); - Timestamp timestamp = Timestamp(0); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(payload).At(timestamp)); - - for (auto _ : state) { - ASSERT_TRUE(runner.Run().ok()); - } - - const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0); - const Matrix& output_matrix = output.packets[0].Get(); - LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x" - << output_matrix.cols(); - LOG(INFO) << "First values=" << output_matrix(0, 0) << ", " - << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", " - << output_matrix(3, 0); -} - -BENCHMARK(BM_ProcessDC); - -} // anonymous namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.cc b/mediapipe/calculators/audio/stabilized_log_calculator.cc deleted file mode 100644 index 0c697a196..000000000 --- a/mediapipe/calculators/audio/stabilized_log_calculator.cc +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Defines StabilizedLogCalculator. - -#include -#include -#include - -#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/proto_ns.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { - -// Example config: -// node { -// calculator: "StabilizedLogCalculator" -// input_stream: "input_time_series" -// output_stream: "stabilized_log_time_series" -// options { -// [mediapipe.StabilizedLogCalculatorOptions.ext] { -// stabilizer: .00001 -// check_nonnegativity: true -// } -// } -// } -class StabilizedLogCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Input stream with TimeSeriesHeader. - ); - cc->Outputs().Index(0).Set( - // Output stabilized log stream with TimeSeriesHeader. - ); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - StabilizedLogCalculatorOptions stabilized_log_calculator_options = - cc->Options(); - - stabilizer_ = stabilized_log_calculator_options.stabilizer(); - output_scale_ = stabilized_log_calculator_options.output_scale(); - check_nonnegativity_ = - stabilized_log_calculator_options.check_nonnegativity(); - CHECK_GE(stabilizer_, 0.0) - << "stabilizer must be >= 0.0, received a value of " << stabilizer_; - - // If the input packets have a header, propagate the header to the output. - if (!cc->Inputs().Index(0).Header().IsEmpty()) { - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - cc->Outputs().Index(0).SetHeader( - Adopt(new TimeSeriesHeader(input_header))); - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - auto input_matrix = cc->Inputs().Index(0).Get(); - if (input_matrix.array().isNaN().any()) { - return absl::InvalidArgumentError("NaN input to log operation."); - } - if (check_nonnegativity_) { - if (input_matrix.minCoeff() < 0.0) { - return absl::OutOfRangeError("Negative input to log operation."); - } - } - std::unique_ptr output_frame(new Matrix( - output_scale_ * (input_matrix.array() + stabilizer_).log().matrix())); - cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - - private: - float stabilizer_; - bool check_nonnegativity_; - double output_scale_; -}; -REGISTER_CALCULATOR(StabilizedLogCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.proto b/mediapipe/calculators/audio/stabilized_log_calculator.proto deleted file mode 100644 index ef6faa8e5..000000000 --- a/mediapipe/calculators/audio/stabilized_log_calculator.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message StabilizedLogCalculatorOptions { - extend CalculatorOptions { - optional StabilizedLogCalculatorOptions ext = 101978339; - } - - // The calculator computes log(x + stabilizer). stabilizer must be >= - // 0, with 0 indicating a lack of stabilization. - optional float stabilizer = 1 [default = .00001]; - - // If true, CHECK that all input values in are >= 0. If false, the - // code will take the log of the potentially negative input values - // plus the stabilizer. - optional bool check_nonnegativity = 2 [default = true]; - - // Support a fixed multiplicative scaling of the output. - optional double output_scale = 3 [default = 1.0]; -} diff --git a/mediapipe/calculators/audio/stabilized_log_calculator_test.cc b/mediapipe/calculators/audio/stabilized_log_calculator_test.cc deleted file mode 100644 index e6e0b5c6f..000000000 --- a/mediapipe/calculators/audio/stabilized_log_calculator_test.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include - -#include "Eigen/Core" -#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { - -const float kStabilizer = 0.1; -const int kNumChannels = 3; -const int kNumSamples = 10; - -class StabilizedLogCalculatorTest - : public TimeSeriesCalculatorTest { - protected: - void SetUp() override { - calculator_name_ = "StabilizedLogCalculator"; - options_.set_stabilizer(kStabilizer); - - input_sample_rate_ = 8000.0; - num_input_channels_ = kNumChannels; - num_input_samples_ = kNumSamples; - } - - void RunGraphNoReturn() { MP_ASSERT_OK(RunGraph()); } -}; - -TEST_F(StabilizedLogCalculatorTest, BasicOperation) { - const int kNumPackets = 5; - - InitializeGraph(); - FillInputHeader(); - - std::vector input_data_matrices; - for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) { - const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond; - Matrix input_data_matrix = - Matrix::Random(kNumChannels, kNumSamples).array().abs(); - input_data_matrices.push_back(input_data_matrix); - AppendInputPacket(new Matrix(input_data_matrix), timestamp); - } - - MP_ASSERT_OK(RunGraph()); - ExpectOutputHeaderEqualsInputHeader(); - for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) { - ExpectApproximatelyEqual( - (input_data_matrices[output_packet].array() + kStabilizer).log(), - runner_->Outputs().Index(0).packets[output_packet].Get()); - } -} - -TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) { - const int kNumPackets = 5; - double output_scale = 2.5; - options_.set_output_scale(output_scale); - - InitializeGraph(); - FillInputHeader(); - - std::vector input_data_matrices; - for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) { - const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond; - Matrix input_data_matrix = - Matrix::Random(kNumChannels, kNumSamples).array().abs(); - input_data_matrices.push_back(input_data_matrix); - AppendInputPacket(new Matrix(input_data_matrix), timestamp); - } - - MP_ASSERT_OK(RunGraph()); - ExpectOutputHeaderEqualsInputHeader(); - for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) { - ExpectApproximatelyEqual( - output_scale * - ((input_data_matrices[output_packet].array() + kStabilizer).log()), - runner_->Outputs().Index(0).packets[output_packet].Get()); - } -} - -TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) { - InitializeGraph(); - FillInputHeader(); - AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)), - 0 /* timestamp */); - MP_ASSERT_OK(RunGraph()); - ExpectOutputHeaderEqualsInputHeader(); - ExpectApproximatelyEqual( - Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(), - runner_->Outputs().Index(0).packets[0].Get()); -} - -TEST_F(StabilizedLogCalculatorTest, NanValuesReturnError) { - InitializeGraph(); - FillInputHeader(); - AppendInputPacket( - new Matrix(Matrix::Constant(kNumChannels, kNumSamples, std::nanf(""))), - 0 /* timestamp */); - ASSERT_FALSE(RunGraph().ok()); -} - -TEST_F(StabilizedLogCalculatorTest, NegativeValuesReturnError) { - InitializeGraph(); - FillInputHeader(); - AppendInputPacket( - new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)), - 0 /* timestamp */); - ASSERT_FALSE(RunGraph().ok()); -} - -TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) { - options_.set_check_nonnegativity(false); - InitializeGraph(); - FillInputHeader(); - AppendInputPacket( - new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)), - 0 /* timestamp */); - MP_ASSERT_OK(RunGraph()); - // Results are undefined. -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/testdata/BUILD b/mediapipe/calculators/audio/testdata/BUILD deleted file mode 100644 index ae679d029..000000000 --- a/mediapipe/calculators/audio/testdata/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -licenses(["notice"]) - -filegroup( - name = "test_audios", - srcs = [ - "sine_wave_1k_44100_mono_2_sec_wav.audio", - "sine_wave_1k_44100_stereo_2_sec_aac.audio", - "sine_wave_1k_44100_stereo_2_sec_mp3.audio", - "sine_wave_1k_48000_stereo_2_sec_wav.audio", - ], - visibility = ["//visibility:public"], -) diff --git a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_mono_2_sec_wav.audio b/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_mono_2_sec_wav.audio deleted file mode 100644 index bd04691d4..000000000 Binary files a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_mono_2_sec_wav.audio and /dev/null differ diff --git a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio b/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio deleted file mode 100644 index 9b3b03a36..000000000 Binary files a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_aac.audio and /dev/null differ diff --git a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio b/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio deleted file mode 100644 index fe11e1165..000000000 Binary files a/mediapipe/calculators/audio/testdata/sine_wave_1k_44100_stereo_2_sec_mp3.audio and /dev/null differ diff --git a/mediapipe/calculators/audio/testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio b/mediapipe/calculators/audio/testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio deleted file mode 100644 index 4258bc4c2..000000000 Binary files a/mediapipe/calculators/audio/testdata/sine_wave_1k_48000_stereo_2_sec_wav.audio and /dev/null differ diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.cc b/mediapipe/calculators/audio/time_series_framer_calculator.cc deleted file mode 100644 index fbbf34226..000000000 --- a/mediapipe/calculators/audio/time_series_framer_calculator.cc +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Defines TimeSeriesFramerCalculator. -#include - -#include -#include -#include - -#include "Eigen/Core" -#include "audio/dsp/window_functions.h" -#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { - -// MediaPipe Calculator for framing a (vector-valued) input time series, -// i.e. for breaking an input time series into fixed-size, possibly -// overlapping, frames. The output stream's frame duration is -// specified by frame_duration_seconds in the -// TimeSeriesFramerCalculatorOptions, and the output's overlap is -// specified by frame_overlap_seconds. -// -// This calculator assumes that the input timestamps refer to the -// first sample in each Matrix. The output timestamps follow this -// same convention. -// -// All output frames will have exactly the same number of samples: the number of -// samples that approximates frame_duration_seconds most closely. -// -// Similarly, frame overlap is by default the (fixed) number of samples -// approximating frame_overlap_seconds most closely. But if -// emulate_fractional_frame_overlap is set to true, frame overlap is a variable -// number of samples instead, such that the long-term average step between -// frames is the difference between the (nominal) frame_duration_seconds and -// frame_overlap_seconds. -// -// If pad_final_packet is true, all input samples will be emitted and the final -// packet will be zero padded as necessary. If pad_final_packet is false, some -// samples may be dropped at the end of the stream. -// -// If use_local_timestamp is true, the output packet's timestamp is based on the -// last sample of the packet. The timestamp of this sample is inferred by -// input_packet_timesamp + local_sample_index / sampling_rate_. If false, the -// output packet's timestamp is based on the cumulative timestamping, which is -// done by adopting the timestamp of the first sample of the packet and this -// sample's timestamp is inferred by initial_input_timestamp_ + -// cumulative_completed_samples / sample_rate_. -class TimeSeriesFramerCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set( - // Input stream with TimeSeriesHeader. - ); - cc->Outputs().Index(0).Set( - // Fixed length time series Packets with TimeSeriesHeader. - ); - return absl::OkStatus(); - } - - // Returns FAIL if the input stream header is invalid. - absl::Status Open(CalculatorContext* cc) override; - - // Outputs as many framed packets as possible given the accumulated - // input. Always returns OK. - absl::Status Process(CalculatorContext* cc) override; - - // Flushes any remaining samples in a zero-padded packet. Always - // returns OK. - absl::Status Close(CalculatorContext* cc) override; - - private: - // Adds input data to the internal buffer. - void EnqueueInput(CalculatorContext* cc); - // Constructs and emits framed output packets. - void FrameOutput(CalculatorContext* cc); - - Timestamp CurrentOutputTimestamp() { - if (use_local_timestamp_) { - return current_timestamp_; - } - return CumulativeOutputTimestamp(); - } - - Timestamp CumulativeOutputTimestamp() { - return initial_input_timestamp_ + - round(cumulative_completed_samples_ / sample_rate_ * - Timestamp::kTimestampUnitsPerSecond); - } - - // Returns the timestamp of a sample on a base, which is usually the time - // stamp of a packet. - Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base, - int64 number_of_samples) { - return timestamp_base + round(number_of_samples / sample_rate_ * - Timestamp::kTimestampUnitsPerSecond); - } - - // The number of input samples to advance after the current output frame is - // emitted. - int next_frame_step_samples() const { - // All numbers are in input samples. - const int64 current_output_frame_start = static_cast( - round(cumulative_output_frames_ * average_frame_step_samples_)); - CHECK_EQ(current_output_frame_start, cumulative_completed_samples_); - const int64 next_output_frame_start = static_cast( - round((cumulative_output_frames_ + 1) * average_frame_step_samples_)); - return next_output_frame_start - current_output_frame_start; - } - - double sample_rate_; - bool pad_final_packet_; - int frame_duration_samples_; - // The advance, in input samples, between the start of successive output - // frames. This may be a non-integer average value if - // emulate_fractional_frame_overlap is true. - double average_frame_step_samples_; - int samples_still_to_drop_; - int64 cumulative_output_frames_; - // "Completed" samples are samples that are no longer needed because - // the framer has completely stepped past them (taking into account - // any overlap). - int64 cumulative_completed_samples_; - Timestamp initial_input_timestamp_; - // The current timestamp is updated along with the incoming packets. - Timestamp current_timestamp_; - int num_channels_; - - // Each entry in this deque consists of a single sample, i.e. a - // single column vector, and its timestamp. - std::deque> sample_buffer_; - - bool use_window_; - Matrix window_; - - bool use_local_timestamp_; -}; -REGISTER_CALCULATOR(TimeSeriesFramerCalculator); - -void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) { - const Matrix& input_frame = cc->Inputs().Index(0).Get(); - - for (int i = 0; i < input_frame.cols(); ++i) { - sample_buffer_.emplace_back(std::make_pair( - input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i))); - } -} - -void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) { - while (sample_buffer_.size() >= - frame_duration_samples_ + samples_still_to_drop_) { - while (samples_still_to_drop_ > 0) { - sample_buffer_.pop_front(); - --samples_still_to_drop_; - } - const int frame_step_samples = next_frame_step_samples(); - std::unique_ptr output_frame( - new Matrix(num_channels_, frame_duration_samples_)); - for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_); - ++i) { - output_frame->col(i) = sample_buffer_.front().first; - current_timestamp_ = sample_buffer_.front().second; - sample_buffer_.pop_front(); - } - const int frame_overlap_samples = - frame_duration_samples_ - frame_step_samples; - if (frame_overlap_samples > 0) { - for (int i = 0; i < frame_overlap_samples; ++i) { - output_frame->col(i + frame_step_samples) = sample_buffer_[i].first; - current_timestamp_ = sample_buffer_[i].second; - } - } else { - samples_still_to_drop_ = -frame_overlap_samples; - } - - if (use_window_) { - *output_frame = (output_frame->array() * window_.array()).matrix(); - } - - cc->Outputs().Index(0).Add(output_frame.release(), - CurrentOutputTimestamp()); - ++cumulative_output_frames_; - cumulative_completed_samples_ += frame_step_samples; - } - if (!use_local_timestamp_) { - // In non-local timestamp mode the timestamp of the next packet will be - // equal to CumulativeOutputTimestamp(). Inform the framework about this - // fact to enable packet queueing optimizations. - cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp()); - } -} - -absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) { - if (initial_input_timestamp_ == Timestamp::Unstarted()) { - initial_input_timestamp_ = cc->InputTimestamp(); - current_timestamp_ = initial_input_timestamp_; - } - - EnqueueInput(cc); - FrameOutput(cc); - - return absl::OkStatus(); -} - -absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) { - while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) { - sample_buffer_.pop_front(); - --samples_still_to_drop_; - } - if (!sample_buffer_.empty() && pad_final_packet_) { - std::unique_ptr output_frame(new Matrix); - output_frame->setZero(num_channels_, frame_duration_samples_); - for (int i = 0; i < sample_buffer_.size(); ++i) { - output_frame->col(i) = sample_buffer_[i].first; - current_timestamp_ = sample_buffer_[i].second; - } - - cc->Outputs().Index(0).Add(output_frame.release(), - CurrentOutputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) { - TimeSeriesFramerCalculatorOptions framer_options = - cc->Options(); - - RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0) - << "Invalid or missing frame_duration_seconds. " - << "framer_duration_seconds: \n" - << framer_options.frame_duration_seconds(); - RET_CHECK_LT(framer_options.frame_overlap_seconds(), - framer_options.frame_duration_seconds()) - << "Invalid frame_overlap_seconds. framer_overlap_seconds: \n" - << framer_options.frame_overlap_seconds(); - - TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), &input_header)); - - sample_rate_ = input_header.sample_rate(); - num_channels_ = input_header.num_channels(); - frame_duration_samples_ = time_series_util::SecondsToSamples( - framer_options.frame_duration_seconds(), sample_rate_); - RET_CHECK_GT(frame_duration_samples_, 0) - << "Frame duration of " << framer_options.frame_duration_seconds() - << "s too small to cover a single sample at " << sample_rate_ << " Hz "; - if (framer_options.emulate_fractional_frame_overlap()) { - // Frame step may be fractional. - average_frame_step_samples_ = (framer_options.frame_duration_seconds() - - framer_options.frame_overlap_seconds()) * - sample_rate_; - } else { - // Frame step is an integer (stored in a double). - average_frame_step_samples_ = - frame_duration_samples_ - - time_series_util::SecondsToSamples( - framer_options.frame_overlap_seconds(), sample_rate_); - } - RET_CHECK_GE(average_frame_step_samples_, 1) - << "Frame step too small to cover a single sample at " << sample_rate_ - << " Hz."; - pad_final_packet_ = framer_options.pad_final_packet(); - - auto output_header = new TimeSeriesHeader(input_header); - output_header->set_num_samples(frame_duration_samples_); - if (round(average_frame_step_samples_) == average_frame_step_samples_) { - // Only set output packet rate if it is fixed. - output_header->set_packet_rate(sample_rate_ / average_frame_step_samples_); - } - cc->Outputs().Index(0).SetHeader(Adopt(output_header)); - cumulative_completed_samples_ = 0; - cumulative_output_frames_ = 0; - samples_still_to_drop_ = 0; - initial_input_timestamp_ = Timestamp::Unstarted(); - current_timestamp_ = Timestamp::Unstarted(); - - std::vector window_vector; - use_window_ = false; - switch (framer_options.window_function()) { - case TimeSeriesFramerCalculatorOptions::HAMMING: - audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_, - &window_vector); - use_window_ = true; - break; - case TimeSeriesFramerCalculatorOptions::HANN: - audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_, - &window_vector); - use_window_ = true; - break; - case TimeSeriesFramerCalculatorOptions::NONE: - break; - } - - if (use_window_) { - window_ = Matrix::Ones(num_channels_, 1) * - Eigen::Map(window_vector.data(), 1, - frame_duration_samples_) - .cast(); - } - use_local_timestamp_ = framer_options.use_local_timestamp(); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.proto b/mediapipe/calculators/audio/time_series_framer_calculator.proto deleted file mode 100644 index 9e5b07462..000000000 --- a/mediapipe/calculators/audio/time_series_framer_calculator.proto +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TimeSeriesFramerCalculatorOptions { - extend CalculatorOptions { - optional TimeSeriesFramerCalculatorOptions ext = 50631621; - } - - // Frame duration in seconds. Required. Must be greater than 0. This is - // rounded to the nearest integer number of samples. - optional double frame_duration_seconds = 1; - - // Frame overlap in seconds. - // - // If emulate_fractional_frame_overlap is false (the default), then the frame - // overlap is rounded to the nearest integer number of samples, and the step - // from one frame to the next will be the difference between the number of - // samples in a frame and the number of samples in the overlap. - // - // If emulate_fractional_frame_overlap is true, then frame overlap will be a - // variable number of samples, such that the long-time average time step from - // one frame to the next will be the difference between the (nominal, not - // rounded) frame_duration_seconds and frame_overlap_seconds. This is useful - // where the desired time step is not an integral number of input samples. - // - // A negative frame_overlap_seconds corresponds to skipping some input samples - // between each frame of emitted samples. - // - // Required that frame_overlap_seconds < frame_duration_seconds. - optional double frame_overlap_seconds = 2 [default = 0.0]; - - // See frame_overlap_seconds for semantics. - optional bool emulate_fractional_frame_overlap = 5 [default = false]; - - // Whether to pad the final packet with zeros. If true, guarantees that all - // input samples (other than those that fall in gaps implied by negative - // frame_overlap_seconds) will be emitted. If set to false, any partial - // packet at the end of the stream will be dropped. - optional bool pad_final_packet = 3 [default = true]; - - // Optional windowing function. The default is NONE (no windowing function). - enum WindowFunction { - NONE = 0; - HAMMING = 1; - HANN = 2; - } - optional WindowFunction window_function = 4 [default = NONE]; - - // If use_local_timestamp is true, the output packet's timestamp is based on - // the last sample of the packet and it's inferred from the latest input - // packet's timestamp. If false, the output packet's timestamp is based on - // the cumulative timestamping, which is inferred from the intial input - // timestamp and the cumulative number of samples. - optional bool use_local_timestamp = 6 [default = false]; -} diff --git a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc b/mediapipe/calculators/audio/time_series_framer_calculator_test.cc deleted file mode 100644 index ca88cebb5..000000000 --- a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include - -#include "Eigen/Core" -#include "audio/dsp/window_functions.h" -#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/time_series_test_util.h" - -namespace mediapipe { -namespace { - -const int kInitialTimestampOffsetMicroseconds = 4; -const int kGapBetweenPacketsInSeconds = 1; -const int kUniversalInputPacketSize = 50; - -class TimeSeriesFramerCalculatorTest - : public TimeSeriesCalculatorTest { - protected: - void SetUp() override { - calculator_name_ = "TimeSeriesFramerCalculator"; - input_sample_rate_ = 4000.0; - num_input_channels_ = 3; - } - - // Returns a float value with the channel and timestamp separated by - // an order of magnitude, for easy parsing by humans. - float TestValue(int64 timestamp_in_microseconds, int channel) { - return timestamp_in_microseconds + channel / 10.0; - } - - // Caller takes ownership of the returned value. - Matrix* NewTestFrame(int num_channels, int num_samples, - double starting_timestamp_seconds) { - auto matrix = new Matrix(num_channels, num_samples); - for (int c = 0; c < num_channels; ++c) { - for (int i = 0; i < num_samples; ++i) { - int64 timestamp = time_series_util::SecondsToSamples( - starting_timestamp_seconds + i / input_sample_rate_, - Timestamp::kTimestampUnitsPerSecond); - (*matrix)(c, i) = TestValue(timestamp, c); - } - } - return matrix; - } - - // Initializes and runs the test graph. - absl::Status Run() { - InitializeGraph(); - - FillInputHeader(); - InitializeInput(); - - return RunGraph(); - } - - // Creates test input and saves a reference copy. - void InitializeInput() { - concatenated_input_samples_.resize(0, num_input_channels_); - num_input_samples_ = 0; - for (int i = 0; i < 10; ++i) { - // This range of packet sizes was chosen such that some input - // packets will be smaller than the output packet size and other - // input packets will be larger. - int packet_size = (i + 1) * 20; - double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 + - num_input_samples_ / input_sample_rate_; - - Matrix* data_frame = - NewTestFrame(num_input_channels_, packet_size, timestamp_seconds); - - // Keep a reference copy of the input. - // - // conservativeResize() is needed here to preserve the existing - // data. Eigen's resize() resizes without preserving data. - concatenated_input_samples_.conservativeResize( - num_input_channels_, num_input_samples_ + packet_size); - concatenated_input_samples_.rightCols(packet_size) = *data_frame; - num_input_samples_ += packet_size; - - AppendInputPacket(data_frame, round(timestamp_seconds * - Timestamp::kTimestampUnitsPerSecond)); - } - - const int frame_duration_samples = FrameDurationSamples(); - std::vector window_vector; - switch (options_.window_function()) { - case TimeSeriesFramerCalculatorOptions::HAMMING: - audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples, - &window_vector); - break; - case TimeSeriesFramerCalculatorOptions::HANN: - audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples, - &window_vector); - break; - case TimeSeriesFramerCalculatorOptions::NONE: - window_vector.assign(frame_duration_samples, 1.0f); - break; - } - - window_ = Matrix::Ones(num_input_channels_, 1) * - Eigen::Map(window_vector.data(), 1, - frame_duration_samples) - .cast(); - } - - int FrameDurationSamples() { - return time_series_util::SecondsToSamples(options_.frame_duration_seconds(), - input_sample_rate_); - } - - // Checks that the values in the framed output packets matches the - // appropriate values from the input. - void CheckOutputPacketValues(const Matrix& actual, int packet_num, - int frame_duration_samples, - double frame_step_samples, - int num_columns_to_check) { - ASSERT_EQ(frame_duration_samples, actual.cols()); - Matrix expected = (concatenated_input_samples_ - .block(0, round(frame_step_samples * packet_num), - num_input_channels_, num_columns_to_check) - .array() * - window_.leftCols(num_columns_to_check).array()) - .matrix(); - ExpectApproximatelyEqual(expected, actual.leftCols(num_columns_to_check)); - } - - // Checks output headers, Timestamps, and values. - void CheckOutput() { - const int frame_duration_samples = FrameDurationSamples(); - const double frame_step_samples = - options_.emulate_fractional_frame_overlap() - ? (options_.frame_duration_seconds() - - options_.frame_overlap_seconds()) * - input_sample_rate_ - : frame_duration_samples - - time_series_util::SecondsToSamples( - options_.frame_overlap_seconds(), input_sample_rate_); - - TimeSeriesHeader expected_header = input().header.Get(); - expected_header.set_num_samples(frame_duration_samples); - if (!options_.emulate_fractional_frame_overlap() || - frame_step_samples == round(frame_step_samples)) { - expected_header.set_packet_rate(input_sample_rate_ / frame_step_samples); - } - ExpectOutputHeaderEquals(expected_header); - - int num_full_packets = output().packets.size(); - if (options_.pad_final_packet()) { - num_full_packets -= 1; - } - - for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) { - const Packet& packet = output().packets[packet_num]; - CheckOutputPacketValues(packet.Get(), packet_num, - frame_duration_samples, frame_step_samples, - frame_duration_samples); - } - - // What is the effective time index of the final sample emitted? - // This includes accounting for the gaps when overlap is negative. - const int num_unique_output_samples = - round((output().packets.size() - 1) * frame_step_samples) + - frame_duration_samples; - LOG(INFO) << "packets.size()=" << output().packets.size() - << " frame_duration_samples=" << frame_duration_samples - << " frame_step_samples=" << frame_step_samples - << " num_input_samples_=" << num_input_samples_ - << " num_unique_output_samples=" << num_unique_output_samples; - const int num_padding_samples = - num_unique_output_samples - num_input_samples_; - if (options_.pad_final_packet()) { - EXPECT_LT(num_padding_samples, frame_duration_samples); - // If the input ended during the dropped samples between the end of - // the last emitted frame and where the next one would begin, there - // can be fewer unique output points than input points, even with - // padding. - const int max_dropped_samples = - static_cast(ceil(frame_step_samples - frame_duration_samples)); - EXPECT_GE(num_padding_samples, std::min(0, -max_dropped_samples)); - - if (num_padding_samples > 0) { - // Check the non-padded part of the final packet. - const Matrix& final_matrix = output().packets.back().Get(); - CheckOutputPacketValues(final_matrix, num_full_packets, - frame_duration_samples, frame_step_samples, - frame_duration_samples - num_padding_samples); - // Check the padded part of the final packet. - EXPECT_EQ( - Matrix::Zero(num_input_channels_, num_padding_samples), - final_matrix.block(0, frame_duration_samples - num_padding_samples, - num_input_channels_, num_padding_samples)); - } - } else { - EXPECT_GT(num_padding_samples, -frame_duration_samples); - EXPECT_LE(num_padding_samples, 0); - } - } - - int num_input_samples_; - Matrix concatenated_input_samples_; - Matrix window_; -}; - -TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationNoOverlap) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, - IntegerSampleDurationNoOverlapHammingWindow) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING); - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, - IntegerSampleDurationNoOverlapHannWindow) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN); - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, IntegerSampleDurationAndOverlap) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(40.0 / input_sample_rate_); - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, NonintegerSampleDurationAndOverlap) { - options_.set_frame_duration_seconds(98.5 / input_sample_rate_); - options_.set_frame_overlap_seconds(38.4 / input_sample_rate_); - - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFrames) { - // Negative overlap means to drop samples between frames. - // 100 samples per frame plus a skip of 10 samples will be 10 full frames in - // the 1100 input samples. - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(-10.0 / input_sample_rate_); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 10); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapExactFramesLessSkip) { - // 100 samples per frame plus a skip of 100 samples will be 6 full frames in - // the 1100 input samples. - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 6); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, NegativeOverlapWithPadding) { - // 150 samples per frame plus a skip of 50 samples will require some padding - // on the sixth and last frame given 1100 sample input. - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(-100.0 / input_sample_rate_); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 6); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, FixedFrameOverlap) { - // Frame of 30 samples with step of 11.4 samples (rounded to 11 samples) - // results in ceil((1100 - 30) / 11) + 1 = 99 packets. - options_.set_frame_duration_seconds(30 / input_sample_rate_); - options_.set_frame_overlap_seconds((30.0 - 11.4) / input_sample_rate_); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 99); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameOverlap) { - // Frame of 30 samples with step of 11.4 samples (not rounded) - // results in ceil((1100 - 30) / 11.4) + 1 = 95 packets. - options_.set_frame_duration_seconds(30 / input_sample_rate_); - options_.set_frame_overlap_seconds((30 - 11.4) / input_sample_rate_); - options_.set_emulate_fractional_frame_overlap(true); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 95); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, VariableFrameSkip) { - // Frame of 30 samples with step of 41.4 samples (not rounded) - // results in ceil((1100 - 30) / 41.4) + 1 = 27 packets. - options_.set_frame_duration_seconds(30 / input_sample_rate_); - options_.set_frame_overlap_seconds((30 - 41.4) / input_sample_rate_); - options_.set_emulate_fractional_frame_overlap(true); - MP_ASSERT_OK(Run()); - EXPECT_EQ(output().packets.size(), 27); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, NoFinalPacketPadding) { - options_.set_frame_duration_seconds(98.5 / input_sample_rate_); - options_.set_pad_final_packet(false); - - MP_ASSERT_OK(Run()); - CheckOutput(); -} - -TEST_F(TimeSeriesFramerCalculatorTest, - FrameRateHigherThanSampleRate_FrameDurationTooLow) { - // Try to produce a frame rate 10 times the input sample rate by using a - // a frame duration that is too small and covers only 0.1 samples. - options_.set_frame_duration_seconds(1 / (10 * input_sample_rate_)); - options_.set_frame_overlap_seconds(0.0); - EXPECT_FALSE(Run().ok()); -} - -TEST_F(TimeSeriesFramerCalculatorTest, - FrameRateHigherThanSampleRate_FrameStepTooLow) { - // Try to produce a frame rate 10 times the input sample rate by using - // a frame overlap that is too high and produces frame steps (difference - // between duration and overlap) of 0.1 samples. - options_.set_frame_duration_seconds(10.0 / input_sample_rate_); - options_.set_frame_overlap_seconds(9.9 / input_sample_rate_); - EXPECT_FALSE(Run().ok()); -} - -// A simple test class to do windowing sanity checks. Tests from this -// class input a single packet of all ones, and check the average -// value of the single output packet. This is useful as a sanity check -// that the correct windows are applied. -class TimeSeriesFramerCalculatorWindowingSanityTest - : public TimeSeriesFramerCalculatorTest { - protected: - void SetUp() override { - TimeSeriesFramerCalculatorTest::SetUp(); - num_input_channels_ = 1; - } - - void RunAndTestSinglePacketAverage(float expected_average) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - InitializeGraph(); - FillInputHeader(); - AppendInputPacket(new Matrix(Matrix::Ones(1, FrameDurationSamples())), - kInitialTimestampOffsetMicroseconds); - MP_ASSERT_OK(RunGraph()); - ASSERT_EQ(1, output().packets.size()); - ASSERT_NEAR(expected_average * FrameDurationSamples(), - output().packets[0].Get().sum(), 1e-5); - } -}; - -TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, NoWindowSanityCheck) { - RunAndTestSinglePacketAverage(1.0f); -} - -TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, - HammingWindowSanityCheck) { - options_.set_window_function(TimeSeriesFramerCalculatorOptions::HAMMING); - RunAndTestSinglePacketAverage(0.54f); -} - -TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, HannWindowSanityCheck) { - options_.set_window_function(TimeSeriesFramerCalculatorOptions::HANN); - RunAndTestSinglePacketAverage(0.5f); -} - -// A simple test class that checks the local packet time stamp. This class -// generate a series of packets with and without gaps between packets and tests -// the behavior with cumulative timestamping and local packet timestamping. -class TimeSeriesFramerCalculatorTimestampingTest - : public TimeSeriesFramerCalculatorTest { - protected: - // Creates test input and saves a reference copy. - void InitializeInputForTimeStampingTest() { - concatenated_input_samples_.resize(0, num_input_channels_); - num_input_samples_ = 0; - for (int i = 0; i < 10; ++i) { - // This range of packet sizes was chosen such that some input - // packets will be smaller than the output packet size and other - // input packets will be larger. - int packet_size = kUniversalInputPacketSize; - double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 + - num_input_samples_ / input_sample_rate_; - if (options_.use_local_timestamp()) { - timestamp_seconds += kGapBetweenPacketsInSeconds * i; - } - - Matrix* data_frame = - NewTestFrame(num_input_channels_, packet_size, timestamp_seconds); - - AppendInputPacket(data_frame, round(timestamp_seconds * - Timestamp::kTimestampUnitsPerSecond)); - num_input_samples_ += packet_size; - } - } - - void CheckOutputTimestamps() { - int num_full_packets = output().packets.size(); - if (options_.pad_final_packet()) { - num_full_packets -= 1; - } - - int64 num_samples = 0; - for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) { - const Packet& packet = output().packets[packet_num]; - num_samples += FrameDurationSamples(); - double expected_timestamp = - options_.use_local_timestamp() - ? GetExpectedLocalTimestampForSample(num_samples - 1) - : GetExpectedCumulativeTimestamp(num_samples - 1); - ASSERT_NEAR(packet.Timestamp().Seconds(), expected_timestamp, 1e-10); - } - } - - absl::Status RunTimestampTest() { - InitializeGraph(); - InitializeInputForTimeStampingTest(); - FillInputHeader(); - return RunGraph(); - } - - private: - // Returns the timestamp in seconds based on local timestamping. - double GetExpectedLocalTimestampForSample(int sample_index) { - return kInitialTimestampOffsetMicroseconds * 1.0e-6 + - sample_index / input_sample_rate_ + - (sample_index / kUniversalInputPacketSize) * - kGapBetweenPacketsInSeconds; - } - - // Returns the timestamp inseconds based on cumulative timestamping. - double GetExpectedCumulativeTimestamp(int sample_index) { - return kInitialTimestampOffsetMicroseconds * 1.0e-6 + - sample_index / FrameDurationSamples() * FrameDurationSamples() / - input_sample_rate_; - } -}; - -TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseLocalTimeStamp) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_use_local_timestamp(true); - - MP_ASSERT_OK(RunTimestampTest()); - CheckOutputTimestamps(); -} - -TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseCumulativeTimeStamp) { - options_.set_frame_duration_seconds(100.0 / input_sample_rate_); - options_.set_use_local_timestamp(false); - - MP_ASSERT_OK(RunTimestampTest()); - CheckOutputTimestamps(); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD deleted file mode 100644 index 0c9dbcd99..000000000 --- a/mediapipe/calculators/core/BUILD +++ /dev/null @@ -1,1185 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -mediapipe_proto_library( - name = "concatenate_vector_calculator_proto", - srcs = ["concatenate_vector_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "dequantize_byte_array_calculator_proto", - srcs = ["dequantize_byte_array_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "packet_cloner_calculator_proto", - srcs = ["packet_cloner_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "packet_resampler_calculator_proto", - srcs = ["packet_resampler_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "packet_thinner_calculator_proto", - srcs = ["packet_thinner_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "split_vector_calculator_proto", - srcs = ["split_vector_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "quantize_float_vector_calculator_proto", - srcs = ["quantize_float_vector_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "sequence_shift_calculator_proto", - srcs = ["sequence_shift_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "gate_calculator_proto", - srcs = ["gate_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "constant_side_packet_calculator_proto", - srcs = ["constant_side_packet_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/framework/formats:classification_proto", - ], -) - -mediapipe_proto_library( - name = "clip_vector_size_calculator_proto", - srcs = ["clip_vector_size_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "flow_limiter_calculator_proto", - srcs = ["flow_limiter_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "add_header_calculator", - srcs = ["add_header_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "add_header_calculator_test", - size = "small", - srcs = ["add_header_calculator_test.cc"], - deps = [ - ":add_header_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_library( - name = "begin_loop_calculator", - srcs = ["begin_loop_calculator.cc"], - hdrs = ["begin_loop_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework:packet", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], - alwayslink = 1, -) - -cc_library( - name = "end_loop_calculator", - srcs = ["end_loop_calculator.cc"], - hdrs = ["end_loop_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework:packet", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:render_data_cc_proto", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -cc_test( - name = "begin_end_loop_calculator_graph_test", - srcs = ["begin_end_loop_calculator_graph_test.cc"], - deps = [ - ":begin_loop_calculator", - ":end_loop_calculator", - ":gate_calculator", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "concatenate_vector_calculator_hdr", - hdrs = ["concatenate_vector_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":concatenate_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/api2:port", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "concatenate_vector_calculator", - srcs = ["concatenate_vector_calculator.cc"], - hdrs = ["concatenate_vector_calculator.h"], - copts = select({ - # Needed for "//mediapipe/framework/formats:tensor" compatibility on Apple - # platforms for Metal pulled in via the tensor.h header. - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":concatenate_vector_calculator_cc_proto", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/api2:port", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework:calculator_framework", - "//mediapipe/util:render_data_cc_proto", - "@org_tensorflow//tensorflow/lite:framework", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//mediapipe:ios": [], - "//conditions:default": [ - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "concatenate_detection_vector_calculator", - srcs = ["concatenate_detection_vector_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":concatenate_vector_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - ], - alwayslink = 1, -) - -cc_library( - name = "concatenate_normalized_landmark_list_calculator", - srcs = ["concatenate_normalized_landmark_list_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":concatenate_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "concatenate_normalized_landmark_list_calculator_test", - srcs = ["concatenate_normalized_landmark_list_calculator_test.cc"], - deps = [ - ":concatenate_normalized_landmark_list_calculator", - ":concatenate_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "concatenate_vector_calculator_test", - srcs = ["concatenate_vector_calculator_test.cc"], - deps = [ - ":concatenate_vector_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "clip_vector_size_calculator", - srcs = ["clip_vector_size_calculator.cc"], - hdrs = ["clip_vector_size_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":clip_vector_size_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "clip_detection_vector_size_calculator", - srcs = ["clip_detection_vector_size_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":clip_vector_size_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - ], - alwayslink = 1, -) - -cc_test( - name = "clip_vector_size_calculator_test", - srcs = ["clip_vector_size_calculator_test.cc"], - deps = [ - ":clip_vector_size_calculator", - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "counting_source_calculator", - srcs = ["counting_source_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "make_pair_calculator", - srcs = ["make_pair_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "matrix_multiply_calculator", - srcs = ["matrix_multiply_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:status", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "matrix_subtract_calculator", - srcs = ["matrix_subtract_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:status", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "mux_calculator", - srcs = ["mux_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/stream_handler:mux_input_stream_handler", - ], - alwayslink = 1, -) - -cc_library( - name = "non_zero_calculator", - srcs = ["non_zero_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -cc_test( - name = "non_zero_calculator_test", - size = "small", - srcs = ["non_zero_calculator_test.cc"], - deps = [ - ":non_zero_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_test( - name = "mux_calculator_test", - srcs = ["mux_calculator_test.cc"], - deps = [ - ":mux_calculator", - ":round_robin_demux_calculator", - ":split_vector_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "packet_cloner_calculator", - srcs = ["packet_cloner_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":packet_cloner_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "packet_inner_join_calculator", - srcs = ["packet_inner_join_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_inner_join_calculator_test", - srcs = ["packet_inner_join_calculator_test.cc"], - deps = [ - ":packet_inner_join_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_library( - name = "packet_thinner_calculator", - srcs = ["packet_thinner_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:options_util", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_thinner_calculator_test", - srcs = ["packet_thinner_calculator_test.cc"], - deps = [ - ":packet_thinner_calculator", - "//mediapipe/calculators/core:packet_thinner_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "pass_through_calculator", - srcs = ["pass_through_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "round_robin_demux_calculator", - srcs = ["round_robin_demux_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -cc_library( - name = "immediate_mux_calculator", - srcs = ["immediate_mux_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "packet_presence_calculator", - srcs = ["packet_presence_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_presence_calculator_test", - srcs = ["packet_presence_calculator_test.cc"], - deps = [ - ":gate_calculator", - ":packet_presence_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - ], -) - -cc_library( - name = "previous_loopback_calculator", - srcs = ["previous_loopback_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - ], - alwayslink = 1, -) - -cc_library( - name = "flow_limiter_calculator", - srcs = ["flow_limiter_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":flow_limiter_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/util:header_util", - ], - alwayslink = 1, -) - -cc_library( - name = "string_to_int_calculator", - srcs = ["string_to_int_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "default_side_packet_calculator", - srcs = ["default_side_packet_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "side_packet_to_stream_calculator", - srcs = ["side_packet_to_stream_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "side_packet_to_stream_calculator_test", - srcs = ["side_packet_to_stream_calculator_test.cc"], - deps = [ - ":side_packet_to_stream_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:options_util", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "immediate_mux_calculator_test", - srcs = ["immediate_mux_calculator_test.cc"], - deps = [ - ":immediate_mux_calculator", - ":round_robin_demux_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:test_calculators", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:threadpool", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "packet_resampler_calculator", - srcs = ["packet_resampler_calculator.cc"], - hdrs = ["packet_resampler_calculator.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework/deps:mathutil", - "//mediapipe/framework/deps:random", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:options_util", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_resampler_calculator_test", - timeout = "short", - srcs = [ - "packet_resampler_calculator_test.cc", - ], - deps = [ - ":packet_resampler_calculator", - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "previous_loopback_calculator_test", - srcs = ["previous_loopback_calculator_test.cc"], - deps = [ - ":gate_calculator", - ":make_pair_calculator", - ":pass_through_calculator", - ":previous_loopback_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "matrix_multiply_calculator_test", - srcs = ["matrix_multiply_calculator_test.cc"], - visibility = ["//visibility:private"], - deps = [ - ":matrix_multiply_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/tool:validate_type", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "matrix_subtract_calculator_test", - srcs = ["matrix_subtract_calculator_test.cc"], - visibility = ["//visibility:private"], - deps = [ - ":matrix_subtract_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@eigen_archive//:eigen3", - ], -) - -cc_test( - name = "flow_limiter_calculator_test", - srcs = ["flow_limiter_calculator_test.cc"], - deps = [ - ":flow_limiter_calculator", - ":flow_limiter_calculator_cc_proto", - "//mediapipe/calculators/core:counting_source_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:test_calculators", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/tool:simulation_clock", - "//mediapipe/framework/tool:simulation_clock_executor", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/time", - ], -) - -cc_library( - name = "split_vector_calculator", - srcs = ["split_vector_calculator.cc"], - hdrs = ["split_vector_calculator.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":split_vector_calculator_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:resource_util", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//mediapipe:ios": [], - "//conditions:default": [ - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - ], - }), - alwayslink = 1, -) - -cc_test( - name = "split_vector_calculator_test", - srcs = ["split_vector_calculator_test.cc"], - deps = [ - ":split_vector_calculator", - ":split_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - ], -) - -cc_library( - name = "split_normalized_landmark_list_calculator", - srcs = ["split_normalized_landmark_list_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":split_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:resource_util", - ], - alwayslink = 1, -) - -cc_test( - name = "split_normalized_landmark_list_calculator_test", - srcs = ["split_normalized_landmark_list_calculator_test.cc"], - deps = [ - ":split_normalized_landmark_list_calculator", - ":split_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_library( - name = "dequantize_byte_array_calculator", - srcs = ["dequantize_byte_array_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":dequantize_byte_array_calculator_cc_proto", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "dequantize_byte_array_calculator_test", - srcs = ["dequantize_byte_array_calculator_test.cc"], - deps = [ - ":dequantize_byte_array_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "quantize_float_vector_calculator", - srcs = ["quantize_float_vector_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":quantize_float_vector_calculator_cc_proto", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "quantize_float_vector_calculator_test", - srcs = ["quantize_float_vector_calculator_test.cc"], - deps = [ - ":quantize_float_vector_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "sequence_shift_calculator", - srcs = ["sequence_shift_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":sequence_shift_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "sequence_shift_calculator_test", - srcs = ["sequence_shift_calculator_test.cc"], - deps = [ - ":sequence_shift_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_library( - name = "gate_calculator", - srcs = ["gate_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":gate_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:header_util", - ], - alwayslink = 1, -) - -cc_test( - name = "gate_calculator_test", - srcs = ["gate_calculator_test.cc"], - deps = [ - ":gate_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_library( - name = "matrix_to_vector_calculator", - srcs = ["matrix_to_vector_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:status_util", - "//mediapipe/util:time_series_util", - "@com_google_absl//absl/memory", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_test( - name = "matrix_to_vector_calculator_test", - srcs = ["matrix_to_vector_calculator_test.cc"], - deps = [ - ":matrix_to_vector_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:validate_type", - "//mediapipe/util:time_series_test_util", - "//mediapipe/util:time_series_util", - ], -) - -cc_library( - name = "merge_calculator", - srcs = ["merge_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "merge_calculator_test", - srcs = ["merge_calculator_test.cc"], - deps = [ - ":merge_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "stream_to_side_packet_calculator", - srcs = ["stream_to_side_packet_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "stream_to_side_packet_calculator_test", - srcs = ["stream_to_side_packet_calculator_test.cc"], - deps = [ - ":stream_to_side_packet_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "constant_side_packet_calculator", - srcs = ["constant_side_packet_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":constant_side_packet_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "constant_side_packet_calculator_test", - srcs = ["constant_side_packet_calculator_test.cc"], - deps = [ - ":constant_side_packet_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) diff --git a/mediapipe/calculators/core/add_header_calculator.cc b/mediapipe/calculators/core/add_header_calculator.cc deleted file mode 100644 index 1c636afd0..000000000 --- a/mediapipe/calculators/core/add_header_calculator.cc +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/logging.h" - -namespace mediapipe { -namespace api2 { - -// Attach the header from a stream or side input to another stream. -// -// The header stream (tag HEADER) must not have any packets in it. -// -// Before using this calculator, please think about changing your -// calculator to not need a header or to accept a separate stream with -// a header, that would be more future proof. -// -// Example usage 1: -// node { -// calculator: "AddHeaderCalculator" -// input_stream: "DATA:audio" -// input_stream: "HEADER:audio_header" -// output_stream: "audio_with_header" -// } -// -// Example usage 2: -// node { -// calculator: "AddHeaderCalculator" -// input_stream: "DATA:audio" -// input_side_packet: "HEADER:audio_header" -// output_stream: "audio_with_header" -// } -// -class AddHeaderCalculator : public Node { - public: - static constexpr Input::Optional kHeader{"HEADER"}; - static constexpr SideInput::Optional kHeaderSide{"HEADER"}; - static constexpr Input kData{"DATA"}; - static constexpr Output> kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kHeader, kHeaderSide, kData, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { - if (kHeader(cc).IsConnected() == kHeaderSide(cc).IsConnected()) { - return absl::InvalidArgumentError( - "Header must be provided via exactly one of side input and input " - "stream"); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const PacketBase& header = - kHeader(cc).IsConnected() ? kHeader(cc).Header() : kHeaderSide(cc); - if (!header.IsEmpty()) { - kOut(cc).SetHeader(header); - } - cc->SetOffset(0); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - kOut(cc).Send(kData(cc).packet()); - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(AddHeaderCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/add_header_calculator_test.cc b/mediapipe/calculators/core/add_header_calculator_test.cc deleted file mode 100644 index 4e197918d..000000000 --- a/mediapipe/calculators/core/add_header_calculator_test.cc +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -class AddHeaderCalculatorTest : public ::testing::Test {}; - -TEST_F(AddHeaderCalculatorTest, HeaderStream) { - CalculatorGraphConfig::Node node; - node.set_calculator("AddHeaderCalculator"); - node.add_input_stream("HEADER:header_stream"); - node.add_input_stream("DATA:data_stream"); - node.add_output_stream("merged_stream"); - - CalculatorRunner runner(node); - - // Set header and add 5 packets. - runner.MutableInputs()->Tag("HEADER").header = - Adopt(new std::string("my_header")); - for (int i = 0; i < 5; ++i) { - Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000)); - runner.MutableInputs()->Tag("DATA").packets.push_back(packet); - } - - // Run calculator. - MP_ASSERT_OK(runner.Run()); - - ASSERT_EQ(1, runner.Outputs().NumEntries()); - - // Test output. - EXPECT_EQ(std::string("my_header"), - runner.Outputs().Index(0).header.Get()); - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(5, output_packets.size()); - for (int i = 0; i < 5; ++i) { - const int val = output_packets[i].Get(); - EXPECT_EQ(i, val); - EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp()); - } -} - -TEST_F(AddHeaderCalculatorTest, HandlesEmptyHeaderStream) { - CalculatorGraphConfig::Node node; - node.set_calculator("AddHeaderCalculator"); - node.add_input_stream("HEADER:header_stream"); - node.add_input_stream("DATA:data_stream"); - node.add_output_stream("merged_stream"); - - CalculatorRunner runner(node); - - // No header and no packets. - // Run calculator. - MP_ASSERT_OK(runner.Run()); - EXPECT_TRUE(runner.Outputs().Index(0).header.IsEmpty()); -} - -TEST_F(AddHeaderCalculatorTest, NoPacketsOnHeaderStream) { - CalculatorGraphConfig::Node node; - node.set_calculator("AddHeaderCalculator"); - node.add_input_stream("HEADER:header_stream"); - node.add_input_stream("DATA:data_stream"); - node.add_output_stream("merged_stream"); - - CalculatorRunner runner(node); - - // Set header and add 5 packets. - runner.MutableInputs()->Tag("HEADER").header = - Adopt(new std::string("my_header")); - runner.MutableInputs()->Tag("HEADER").packets.push_back( - Adopt(new std::string("not allowed"))); - for (int i = 0; i < 5; ++i) { - Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000)); - runner.MutableInputs()->Tag("DATA").packets.push_back(packet); - } - - // Run calculator. - ASSERT_FALSE(runner.Run().ok()); -} - -TEST_F(AddHeaderCalculatorTest, InputSidePacket) { - CalculatorGraphConfig::Node node; - node.set_calculator("AddHeaderCalculator"); - node.add_input_stream("DATA:data_stream"); - node.add_output_stream("merged_stream"); - node.add_input_side_packet("HEADER:header"); - - CalculatorRunner runner(node); - - // Set header and add 5 packets. - runner.MutableSidePackets()->Tag("HEADER") = - Adopt(new std::string("my_header")); - for (int i = 0; i < 5; ++i) { - Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000)); - runner.MutableInputs()->Tag("DATA").packets.push_back(packet); - } - - // Run calculator. - MP_ASSERT_OK(runner.Run()); - - ASSERT_EQ(1, runner.Outputs().NumEntries()); - - // Test output. - EXPECT_EQ(std::string("my_header"), - runner.Outputs().Index(0).header.Get()); - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(5, output_packets.size()); - for (int i = 0; i < 5; ++i) { - const int val = output_packets[i].Get(); - EXPECT_EQ(i, val); - EXPECT_EQ(Timestamp(i * 1000), output_packets[i].Timestamp()); - } -} - -TEST_F(AddHeaderCalculatorTest, UsingBothSideInputAndStream) { - CalculatorGraphConfig::Node node; - node.set_calculator("AddHeaderCalculator"); - node.add_input_stream("HEADER:header_stream"); - node.add_input_stream("DATA:data_stream"); - node.add_output_stream("merged_stream"); - node.add_input_side_packet("HEADER:header"); - - CalculatorRunner runner(node); - - // Set both headers and add 5 packets. - runner.MutableSidePackets()->Tag("HEADER") = - Adopt(new std::string("my_header")); - runner.MutableSidePackets()->Tag("HEADER") = - Adopt(new std::string("my_header")); - for (int i = 0; i < 5; ++i) { - Packet packet = Adopt(new int(i)).At(Timestamp(i * 1000)); - runner.MutableInputs()->Tag("DATA").packets.push_back(packet); - } - - // Run should fail because header can only be provided one way. - EXPECT_EQ(runner.Run().code(), absl::InvalidArgumentError("").code()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc b/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc deleted file mode 100644 index b1ebdd086..000000000 --- a/mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/core/begin_loop_calculator.h" -#include "mediapipe/calculators/core/end_loop_calculator.h" -#include "mediapipe/framework/calculator_contract.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { -namespace { - -MATCHER_P2(PacketOfIntsEq, timestamp, value, "") { - Timestamp actual_timestamp = arg.Timestamp(); - const auto& actual_value = arg.template Get>(); - return testing::Value(actual_timestamp, testing::Eq(timestamp)) && - testing::Value(actual_value, testing::ElementsAreArray(value)); -} - -typedef BeginLoopCalculator> BeginLoopIntegerCalculator; -REGISTER_CALCULATOR(BeginLoopIntegerCalculator); - -class IncrementCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const int& input_int = cc->Inputs().Index(0).Get(); - auto output_int = absl::make_unique(input_int + 1); - cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(IncrementCalculator); - -typedef EndLoopCalculator> EndLoopIntegersCalculator; -REGISTER_CALCULATOR(EndLoopIntegersCalculator); - -class BeginEndLoopCalculatorGraphTest : public ::testing::Test { - protected: - void SetUp() override { - auto graph_config = ParseTextProtoOrDie( - R"pb( - num_threads: 4 - input_stream: "ints" - node { - calculator: "BeginLoopIntegerCalculator" - input_stream: "ITERABLE:ints" - output_stream: "ITEM:int" - output_stream: "BATCH_END:timestamp" - } - node { - calculator: "IncrementCalculator" - input_stream: "int" - output_stream: "int_plus_one" - } - node { - calculator: "EndLoopIntegersCalculator" - input_stream: "ITEM:int_plus_one" - input_stream: "BATCH_END:timestamp" - output_stream: "ITERABLE:ints_plus_one" - } - )pb"); - tool::AddVectorSink("ints_plus_one", &graph_config, &output_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_ASSERT_OK(graph_.StartRun({})); - } - - void SendPacketOfInts(Timestamp timestamp, std::vector ints) { - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "ints", MakePacket>(std::move(ints)).At(timestamp))); - } - - CalculatorGraph graph_; - std::vector output_packets_; -}; - -TEST_F(BeginEndLoopCalculatorGraphTest, InputStreamForIterableIsEmpty) { - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - // EndLoopCalc will forward the timestamp bound because there are no packets - // to process. - ASSERT_EQ(0, output_packets_.size()); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) { - SendPacketOfInts(Timestamp(0), {}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - // EndLoopCalc will forward the timestamp bound because there are no elements - // in collection to output. - EXPECT_TRUE(output_packets_.empty()); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) { - Timestamp input_timestamp = Timestamp(0); - SendPacketOfInts(input_timestamp, {0, 1, 2}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - EXPECT_THAT(output_packets_, - testing::ElementsAre( - PacketOfIntsEq(input_timestamp, std::vector{1, 2, 3}))); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) { - Timestamp input_timestamp0 = Timestamp(0); - SendPacketOfInts(input_timestamp0, {0, 1}); - - Timestamp input_timestamp1 = Timestamp(1); - SendPacketOfInts(input_timestamp1, {}); - - Timestamp input_timestamp2 = Timestamp(2); - SendPacketOfInts(input_timestamp2, {2, 3}); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are - // no elements in vector to process. - EXPECT_THAT(output_packets_, - testing::ElementsAre( - PacketOfIntsEq(input_timestamp0, std::vector{1, 2}), - PacketOfIntsEq(input_timestamp2, std::vector{3, 4}))); -} - -// Passes non empty vector through or outputs empty vector in case of timestamp -// bound update. -class PassThroughOrEmptyVectorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->SetProcessTimestampBounds(true); - cc->Inputs().Index(0).Set>(); - cc->Outputs().Index(0).Set>(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (!cc->Inputs().Index(0).IsEmpty()) { - cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value()); - } else { - cc->Outputs().Index(0).AddPacket( - MakePacket>(std::vector()) - .At(cc->InputTimestamp())); - } - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(PassThroughOrEmptyVectorCalculator); - -class BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest - : public ::testing::Test { - protected: - void SetUp() override { - auto graph_config = ParseTextProtoOrDie( - R"pb( - num_threads: 4 - input_stream: "ints" - input_stream: "force_ints_to_be_timestamp_bound_update" - node { - calculator: "GateCalculator" - input_stream: "ints" - input_stream: "DISALLOW:force_ints_to_be_timestamp_bound_update" - output_stream: "ints_passed_through" - } - node { - calculator: "BeginLoopIntegerCalculator" - input_stream: "ITERABLE:ints_passed_through" - output_stream: "ITEM:int" - output_stream: "BATCH_END:timestamp" - } - node { - calculator: "IncrementCalculator" - input_stream: "int" - output_stream: "int_plus_one" - } - node { - calculator: "EndLoopIntegersCalculator" - input_stream: "ITEM:int_plus_one" - input_stream: "BATCH_END:timestamp" - output_stream: "ITERABLE:ints_plus_one" - } - node { - calculator: "PassThroughOrEmptyVectorCalculator" - input_stream: "ints_plus_one" - output_stream: "ints_plus_one_passed_through" - } - )pb"); - tool::AddVectorSink("ints_plus_one_passed_through", &graph_config, - &output_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_ASSERT_OK(graph_.StartRun({})); - } - - void SendPacketOfIntsOrBound(Timestamp timestamp, std::vector ints) { - // All "ints" packets which are empty are forced to be just timestamp - // bound updates for begin loop calculator. - bool force_ints_to_be_timestamp_bound_update = ints.empty(); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "force_ints_to_be_timestamp_bound_update", - MakePacket(force_ints_to_be_timestamp_bound_update) - .At(timestamp))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "ints", MakePacket>(std::move(ints)).At(timestamp))); - } - - CalculatorGraph graph_; - std::vector output_packets_; -}; - -TEST_F(BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest, - SingleEmptyVector) { - SendPacketOfIntsOrBound(Timestamp(0), {}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - EXPECT_THAT(output_packets_, testing::ElementsAre(PacketOfIntsEq( - Timestamp(0), std::vector{}))); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest, - SingleNonEmptyVector) { - SendPacketOfIntsOrBound(Timestamp(0), {0, 1, 2}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - EXPECT_THAT(output_packets_, testing::ElementsAre(PacketOfIntsEq( - Timestamp(0), std::vector{1, 2, 3}))); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphProcessingEmptyPacketsTest, MultipleVectors) { - SendPacketOfIntsOrBound(Timestamp(0), {}); - // Waiting until idle to guarantee all timestamp bound updates are processed - // individually. (Timestamp bounds updates occur in the provide config only - // if input is an empty vector.) - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - SendPacketOfIntsOrBound(Timestamp(1), {0, 1}); - SendPacketOfIntsOrBound(Timestamp(2), {}); - // Waiting until idle to guarantee all timestamp bound updates are processed - // individually. (Timestamp bounds updates occur in the provide config only - // if input is an empty vector.) - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - SendPacketOfIntsOrBound(Timestamp(3), {2, 3}); - SendPacketOfIntsOrBound(Timestamp(4), {}); - // Waiting until idle to guarantee all timestamp bound updates are processed - // individually. (Timestamp bounds updates occur in the provide config only - // if input is an empty vector.) - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - EXPECT_THAT( - output_packets_, - testing::ElementsAre(PacketOfIntsEq(Timestamp(0), std::vector{}), - PacketOfIntsEq(Timestamp(1), std::vector{1, 2}), - PacketOfIntsEq(Timestamp(2), std::vector{}), - PacketOfIntsEq(Timestamp(3), std::vector{3, 4}), - PacketOfIntsEq(Timestamp(4), std::vector{}))); -} - -class MultiplierCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Inputs().Index(1).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const int& input_int = cc->Inputs().Index(0).Get(); - const int& multiplier_int = cc->Inputs().Index(1).Get(); - auto output_int = absl::make_unique(input_int * multiplier_int); - cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(MultiplierCalculator); - -class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test { - protected: - void SetUp() override { - auto graph_config = ParseTextProtoOrDie( - R"pb( - num_threads: 4 - input_stream: "ints" - input_stream: "multiplier" - node { - calculator: "BeginLoopIntegerCalculator" - input_stream: "ITERABLE:ints" - input_stream: "CLONE:multiplier" - output_stream: "ITEM:int_at_loop" - output_stream: "CLONE:multiplier_cloned_at_loop" - output_stream: "BATCH_END:timestamp" - } - node { - calculator: "MultiplierCalculator" - input_stream: "int_at_loop" - input_stream: "multiplier_cloned_at_loop" - output_stream: "multiplied_int_at_loop" - } - node { - calculator: "EndLoopIntegersCalculator" - input_stream: "ITEM:multiplied_int_at_loop" - input_stream: "BATCH_END:timestamp" - output_stream: "ITERABLE:multiplied_ints" - } - )pb"); - tool::AddVectorSink("multiplied_ints", &graph_config, &output_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_ASSERT_OK(graph_.StartRun({})); - } - - void SendPackets(Timestamp timestamp, int multiplier, std::vector ints) { - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "ints", MakePacket>(std::move(ints)).At(timestamp))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "multiplier", MakePacket(multiplier).At(timestamp))); - } - - void SendMultiplier(Timestamp timestamp, int multiplier) { - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "multiplier", MakePacket(multiplier).At(timestamp))); - } - - CalculatorGraph graph_; - std::vector output_packets_; -}; - -TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, - InputStreamForIterableIsEmpty) { - Timestamp input_timestamp = Timestamp(42); - SendMultiplier(input_timestamp, /*multiplier=*/2); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - // EndLoopCalc will forward the timestamp bound because there are no packets - // to process. - ASSERT_EQ(0, output_packets_.size()); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) { - SendPackets(Timestamp(0), /*multiplier=*/2, /*ints=*/{}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - // EndLoopCalc will forward the timestamp bound because there are no elements - // in collection to output. - EXPECT_TRUE(output_packets_.empty()); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) { - Timestamp input_timestamp = Timestamp(42); - SendPackets(input_timestamp, /*multiplier=*/2, /*ints=*/{0, 1, 2}); - MP_ASSERT_OK(graph_.WaitUntilIdle()); - - EXPECT_THAT(output_packets_, - testing::ElementsAre( - PacketOfIntsEq(input_timestamp, std::vector{0, 2, 4}))); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); -} - -TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) { - Timestamp input_timestamp0 = Timestamp(42); - SendPackets(input_timestamp0, /*multiplier=*/2, /*ints=*/{0, 1}); - - Timestamp input_timestamp1 = Timestamp(43); - SendPackets(input_timestamp1, /*multiplier=*/2, /*ints=*/{}); - - Timestamp input_timestamp2 = Timestamp(44); - SendPackets(input_timestamp2, /*multiplier=*/3, /*ints=*/{2, 3}); - - MP_ASSERT_OK(graph_.CloseAllPacketSources()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are - // no elements in vector to process. - EXPECT_THAT(output_packets_, - testing::ElementsAre( - PacketOfIntsEq(input_timestamp0, std::vector{0, 2}), - PacketOfIntsEq(input_timestamp2, std::vector{6, 9}))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc deleted file mode 100644 index 1d0f7824d..000000000 --- a/mediapipe/calculators/core/begin_loop_calculator.cc +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/begin_loop_calculator.h" - -#include - -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -// A calculator to process std::vector. -typedef BeginLoopCalculator> - BeginLoopNormalizedLandmarkListVectorCalculator; -REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkListVectorCalculator); - -// A calculator to process std::vector. -typedef BeginLoopCalculator> - BeginLoopNormalizedRectCalculator; -REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator); - -// A calculator to process std::vector. -typedef BeginLoopCalculator> - BeginLoopDetectionCalculator; -REGISTER_CALCULATOR(BeginLoopDetectionCalculator); - -// A calculator to process std::vector. -typedef BeginLoopCalculator> BeginLoopMatrixCalculator; -REGISTER_CALCULATOR(BeginLoopMatrixCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/begin_loop_calculator.h b/mediapipe/calculators/core/begin_loop_calculator.h deleted file mode 100644 index a9d29e687..000000000 --- a/mediapipe/calculators/core/begin_loop_calculator.h +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ - -#include "absl/memory/memory.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_contract.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Calculator for implementing loops on iterable collections inside a MediaPipe -// graph. -// -// It is designed to be used like: -// -// node { -// calculator: "BeginLoopWithIterableCalculator" -// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts -// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts -// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts -// } -// -// node { -// calculator: "ElementToBlaConverterSubgraph" -// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts -// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts -// } -// -// node { -// calculator: "EndLoopWithOutputCalculator" -// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts -// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts -// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts -// } -// -// Input streams tagged with "CLONE" are cloned to the corresponding output -// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph -// can run multiple times, once per element in the "ITERABLE" for each pakcet -// clone of the packets in the "CLONE" input streams. -template -class BeginLoopCalculator : public CalculatorBase { - using ItemT = typename IterableT::value_type; - - public: - static absl::Status GetContract(CalculatorContract* cc) { - // The below enables processing of timestamp bound updates, and that enables - // correct timestamp propagation by the companion EndLoopCalculator. - // - // For instance, Process() function will be still invoked even if upstream - // calculator has updated timestamp bound for ITERABLE input instead of - // providing actual value. - cc->SetProcessTimestampBounds(true); - - // A non-empty packet in the optional "TICK" input stream wakes up the - // calculator. - // DEPRECATED as timestamp bound updates are processed by default in this - // calculator. - if (cc->Inputs().HasTag("TICK")) { - cc->Inputs().Tag("TICK").SetAny(); - } - - // An iterable collection in the input stream. - RET_CHECK(cc->Inputs().HasTag("ITERABLE")); - cc->Inputs().Tag("ITERABLE").Set(); - - // An element from the collection. - RET_CHECK(cc->Outputs().HasTag("ITEM")); - cc->Outputs().Tag("ITEM").Set(); - - RET_CHECK(cc->Outputs().HasTag("BATCH_END")); - cc->Outputs() - .Tag("BATCH_END") - .Set( - // A flush signal to the corresponding EndLoopCalculator for it to - // emit the aggregated result with the timestamp contained in this - // flush signal packet. - ); - - // Input streams tagged with "CLONE" are cloned to the corresponding - // "CLONE" output streams at loop timestamps. - RET_CHECK(cc->Inputs().NumEntries("CLONE") == - cc->Outputs().NumEntries("CLONE")); - if (cc->Inputs().NumEntries("CLONE") > 0) { - for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) { - cc->Inputs().Get("CLONE", i).SetAny(); - cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i)); - } - } - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - Timestamp last_timestamp = loop_internal_timestamp_; - if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) { - const IterableT& collection = - cc->Inputs().Tag("ITERABLE").template Get(); - for (const auto& item : collection) { - cc->Outputs().Tag("ITEM").AddPacket( - MakePacket(item).At(loop_internal_timestamp_)); - ForwardClonePackets(cc, loop_internal_timestamp_); - ++loop_internal_timestamp_; - } - } - - // The collection was empty and nothing was processed. - if (last_timestamp == loop_internal_timestamp_) { - // Increment loop_internal_timestamp_ because it is used up now. - ++loop_internal_timestamp_; - for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) { - it->SetNextTimestampBound(loop_internal_timestamp_); - } - } - - // The for loop processing the input collection already incremented - // loop_internal_timestamp_. To emit BATCH_END packet along the last - // non-BATCH_END packet, decrement by one. - cc->Outputs() - .Tag("BATCH_END") - .AddPacket(MakePacket(cc->InputTimestamp()) - .At(Timestamp(loop_internal_timestamp_ - 1))); - - return absl::OkStatus(); - } - - private: - void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) { - if (cc->Inputs().NumEntries("CLONE") > 0) { - for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) { - if (!cc->Inputs().Get("CLONE", i).IsEmpty()) { - auto input_packet = cc->Inputs().Get("CLONE", i).Value(); - cc->Outputs() - .Get("CLONE", i) - .AddPacket(std::move(input_packet).At(output_timestamp)); - } - } - } - } - - // Fake timestamps generated per element in collection. - Timestamp loop_internal_timestamp_ = Timestamp(0); -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc b/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc deleted file mode 100644 index 55bcf2feb..000000000 --- a/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/clip_vector_size_calculator.h" -#include "mediapipe/framework/formats/detection.pb.h" - -namespace mediapipe { - -typedef ClipVectorSizeCalculator<::mediapipe::Detection> - ClipDetectionVectorSizeCalculator; -REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.cc b/mediapipe/calculators/core/clip_vector_size_calculator.cc deleted file mode 100644 index 89ac0b9ef..000000000 --- a/mediapipe/calculators/core/clip_vector_size_calculator.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/clip_vector_size_calculator.h" - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect> - ClipNormalizedRectVectorSizeCalculator; -REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator); - -typedef ClipVectorSizeCalculator<::mediapipe::Detection> - ClipDetectionVectorSizeCalculator; -REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.h b/mediapipe/calculators/core/clip_vector_size_calculator.h deleted file mode 100644 index 00de9be7f..000000000 --- a/mediapipe/calculators/core/clip_vector_size_calculator.h +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ - -#include -#include - -#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Clips the size of the input vector of type T to a specified max_vec_size. -// In a graph it will be used as: -// node { -// calculator: "ClipIntVectorSizeCalculator" -// input_stream: "input_vector" -// output_stream: "output_vector" -// options { -// [mediapipe.ClipVectorSizeCalculatorOptions.ext] { -// max_vec_size: 5 -// } -// } -// } -// Optionally, you can pass in a side packet that will override `max_vec_size` -// that is specified in the options. -template -class ClipVectorSizeCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().NumEntries() == 1); - RET_CHECK(cc->Outputs().NumEntries() == 1); - - if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>() - .max_vec_size() < 1) { - return absl::InternalError( - "max_vec_size should be greater than or equal to 1."); - } - - cc->Inputs().Index(0).Set>(); - cc->Outputs().Index(0).Set>(); - // Optional input side packet that determines `max_vec_size`. - if (cc->InputSidePackets().NumEntries() > 0) { - cc->InputSidePackets().Index(0).Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>() - .max_vec_size(); - // Override `max_vec_size` if passed as side packet. - if (cc->InputSidePackets().NumEntries() > 0 && - !cc->InputSidePackets().Index(0).IsEmpty()) { - max_vec_size_ = cc->InputSidePackets().Index(0).Get(); - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (max_vec_size_ < 1) { - return absl::InternalError( - "max_vec_size should be greater than or equal to 1."); - } - if (cc->Inputs().Index(0).IsEmpty()) { - return absl::OkStatus(); - } - - return ClipVectorSize(std::is_copy_constructible(), cc); - } - - template - absl::Status ClipVectorSize(std::true_type, CalculatorContext* cc) { - auto output = absl::make_unique>(); - const std::vector& input_vector = - cc->Inputs().Index(0).Get>(); - if (max_vec_size_ >= input_vector.size()) { - output->insert(output->end(), input_vector.begin(), input_vector.end()); - } else { - for (int i = 0; i < max_vec_size_; ++i) { - output->push_back(input_vector[i]); - } - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - - template - absl::Status ClipVectorSize(std::false_type, CalculatorContext* cc) { - return ConsumeAndClipVectorSize(std::is_move_constructible(), cc); - } - - template - absl::Status ConsumeAndClipVectorSize(std::true_type, CalculatorContext* cc) { - auto output = absl::make_unique>(); - absl::StatusOr>> input_status = - cc->Inputs().Index(0).Value().Consume>(); - - if (input_status.ok()) { - std::unique_ptr> input_vector = - std::move(input_status).value(); - auto begin_it = input_vector->begin(); - auto end_it = input_vector->end(); - if (max_vec_size_ < input_vector->size()) { - end_it = input_vector->begin() + max_vec_size_; - } - output->insert(output->end(), std::make_move_iterator(begin_it), - std::make_move_iterator(end_it)); - } else { - return input_status.status(); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - - template - absl::Status ConsumeAndClipVectorSize(std::false_type, - CalculatorContext* cc) { - return absl::InternalError( - "Cannot copy or move input vectors and clip their size."); - } - - private: - int max_vec_size_ = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.proto b/mediapipe/calculators/core/clip_vector_size_calculator.proto deleted file mode 100644 index 6044f77c8..000000000 --- a/mediapipe/calculators/core/clip_vector_size_calculator.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message ClipVectorSizeCalculatorOptions { - extend CalculatorOptions { - optional ClipVectorSizeCalculatorOptions ext = 274674998; - } - - // Maximum size of output vector. - optional int32 max_vec_size = 1 [default = 1]; -} diff --git a/mediapipe/calculators/core/clip_vector_size_calculator_test.cc b/mediapipe/calculators/core/clip_vector_size_calculator_test.cc deleted file mode 100644 index d9852d5a5..000000000 --- a/mediapipe/calculators/core/clip_vector_size_calculator_test.cc +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/clip_vector_size_calculator.h" - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -typedef ClipVectorSizeCalculator TestClipIntVectorSizeCalculator; -REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator); - -void AddInputVector(const std::vector& input, int64 timestamp, - CalculatorRunner* runner) { - runner->MutableInputs()->Index(0).packets.push_back( - MakePacket>(input).At(Timestamp(timestamp))); -} - -TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestClipIntVectorSizeCalculator" - input_stream: "input_vector" - output_stream: "output_vector" - options { - [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 } - } - )pb"); - CalculatorRunner runner(node_config); - - std::vector input = {}; - AddInputVector(input, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - EXPECT_TRUE(outputs[0].Get>().empty()); -} - -TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestClipIntVectorSizeCalculator" - input_stream: "input_vector" - output_stream: "output_vector" - options { - [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 } - } - )pb"); - CalculatorRunner runner(node_config); - - std::vector input = {0, 1, 2, 3}; - AddInputVector(input, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector& output = outputs[0].Get>(); - EXPECT_EQ(2, output.size()); - std::vector expected_vector = {0, 1}; - EXPECT_EQ(expected_vector, output); -} - -TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestClipIntVectorSizeCalculator" - input_stream: "input_vector" - output_stream: "output_vector" - options { - [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 } - } - )pb"); - CalculatorRunner runner(node_config); - - { - std::vector input = {0, 1, 2, 3}; - AddInputVector(input, /*timestamp=*/1, &runner); - } - { - std::vector input = {2, 3, 4, 5}; - AddInputVector(input, /*timestamp=*/2, &runner); - } - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(2, outputs.size()); - { - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector& output = outputs[0].Get>(); - EXPECT_EQ(3, output.size()); - std::vector expected_vector = {0, 1, 2}; - EXPECT_EQ(expected_vector, output); - } - { - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - const std::vector& output = outputs[1].Get>(); - EXPECT_EQ(3, output.size()); - std::vector expected_vector = {2, 3, 4}; - EXPECT_EQ(expected_vector, output); - } -} - -typedef ClipVectorSizeCalculator> - TestClipUniqueIntPtrVectorSizeCalculator; -REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator); - -TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) { - /* Note: We don't use CalculatorRunner for this test because it keeps copies - * of input packets, so packets sent to the graph don't have sole ownership. - * The test needs to send packets that own the data. - */ - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: "input_vector" - node { - calculator: "TestClipUniqueIntPtrVectorSizeCalculator" - input_stream: "input_vector" - output_stream: "output_vector" - options { - [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 } - } - } - )pb"); - - std::vector outputs; - tool::AddVectorSink("output_vector", &graph_config, &outputs); - - CalculatorGraph graph; - MP_EXPECT_OK(graph.Initialize(graph_config)); - MP_EXPECT_OK(graph.StartRun({})); - - // input1 : {0, 1, 2, 3, 4, 5} - auto input_vector = absl::make_unique>>(6); - for (int i = 0; i < 6; ++i) { - input_vector->at(i) = absl::make_unique(i); - } - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "input_vector", Adopt(input_vector.release()).At(Timestamp(1)))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.CloseAllPacketSources()); - MP_EXPECT_OK(graph.WaitUntilDone()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector>& result = - outputs[0].Get>>(); - EXPECT_EQ(3, result.size()); - for (int i = 0; i < 3; ++i) { - const std::unique_ptr& v = result[i]; - EXPECT_EQ(i, *v); - } -} - -TEST(TestClipIntVectorSizeCalculatorTest, SidePacket) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestClipIntVectorSizeCalculator" - input_stream: "input_vector" - input_side_packet: "max_vec_size" - output_stream: "output_vector" - options { - [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 } - } - )pb"); - CalculatorRunner runner(node_config); - // This should override the default of 1 set in the options. - runner.MutableSidePackets()->Index(0) = Adopt(new int(2)); - std::vector input = {0, 1, 2, 3}; - AddInputVector(input, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector& output = outputs[0].Get>(); - EXPECT_EQ(2, output.size()); - std::vector expected_vector = {0, 1}; - EXPECT_EQ(expected_vector, output); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/concatenate_detection_vector_calculator.cc b/mediapipe/calculators/core/concatenate_detection_vector_calculator.cc deleted file mode 100644 index fd7d324b2..000000000 --- a/mediapipe/calculators/core/concatenate_detection_vector_calculator.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2019-2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/concatenate_vector_calculator.h" -#include "mediapipe/framework/formats/detection.pb.h" - -namespace mediapipe { - -// Example config: -// -// node { -// calculator: "ConcatenateDetectionVectorCalculator" -// input_stream: "detection_vector_1" -// input_stream: "detection_vector_2" -// output_stream: "concatenated_detection_vector" -// } -// -typedef ConcatenateVectorCalculator<::mediapipe::Detection> - ConcatenateDetectionVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateDetectionVectorCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc b/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc deleted file mode 100644 index f0a4043a7..000000000 --- a/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_ // NOLINT -#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_ // NOLINT - -#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace api2 { - -// Concatenates several NormalizedLandmarkList protos following stream index -// order. This class assumes that every input stream contains a -// NormalizedLandmarkList proto object. -class ConcatenateNormalizedLandmarkListCalculator : public Node { - public: - static constexpr Input::Multiple kIn{""}; - static constexpr Output kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK_GE(kIn(cc).Count(), 1); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - only_emit_if_all_present_ = - cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>() - .only_emit_if_all_present(); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (only_emit_if_all_present_) { - for (const auto& input : kIn(cc)) { - if (input.IsEmpty()) return absl::OkStatus(); - } - } - - NormalizedLandmarkList output; - for (const auto& input : kIn(cc)) { - if (input.IsEmpty()) continue; - const NormalizedLandmarkList& list = *input; - for (int j = 0; j < list.landmark_size(); ++j) { - *output.add_landmark() = list.landmark(j); - } - } - kOut(cc).Send(std::move(output)); - return absl::OkStatus(); - } - - private: - bool only_emit_if_all_present_; -}; -MEDIAPIPE_REGISTER_NODE(ConcatenateNormalizedLandmarkListCalculator); - -} // namespace api2 -} // namespace mediapipe - -// NOLINTNEXTLINE -#endif // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_NORMALIZED_LIST_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator_test.cc b/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator_test.cc deleted file mode 100644 index fd116ece7..000000000 --- a/mediapipe/calculators/core/concatenate_normalized_landmark_list_calculator_test.cc +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -constexpr float kLocationValue = 3; - -NormalizedLandmarkList GenerateLandmarks(int landmarks_size, - int value_multiplier) { - NormalizedLandmarkList landmarks; - for (int i = 0; i < landmarks_size; ++i) { - NormalizedLandmark* landmark = landmarks.add_landmark(); - landmark->set_x(value_multiplier * kLocationValue); - landmark->set_y(value_multiplier * kLocationValue); - landmark->set_z(value_multiplier * kLocationValue); - } - return landmarks; -} - -void ValidateCombinedLandmarks( - const std::vector& inputs, - const NormalizedLandmarkList& result) { - int element_id = 0; - int expected_size = 0; - for (int i = 0; i < inputs.size(); ++i) { - const NormalizedLandmarkList& landmarks_i = inputs[i]; - expected_size += landmarks_i.landmark_size(); - for (int j = 0; j < landmarks_i.landmark_size(); ++j) { - const NormalizedLandmark& expected = landmarks_i.landmark(j); - const NormalizedLandmark& got = result.landmark(element_id); - EXPECT_FLOAT_EQ(expected.x(), got.x()); - EXPECT_FLOAT_EQ(expected.y(), got.y()); - EXPECT_FLOAT_EQ(expected.z(), got.z()); - ++element_id; - } - } - EXPECT_EQ(expected_size, result.landmark_size()); -} - -void AddInputLandmarkLists( - const std::vector& input_landmarks_vec, - int64 timestamp, CalculatorRunner* runner) { - for (int i = 0; i < input_landmarks_vec.size(); ++i) { - runner->MutableInputs()->Index(i).packets.push_back( - MakePacket(input_landmarks_vec[i]) - .At(Timestamp(timestamp))); - } -} - -TEST(ConcatenateNormalizedLandmarkListCalculatorTest, EmptyVectorInputs) { - CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - NormalizedLandmarkList empty_list; - std::vector inputs = {empty_list, empty_list, - empty_list}; - AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(0, outputs[0].Get().landmark_size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); -} - -TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneTimestamp) { - CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - NormalizedLandmarkList input_0 = - GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0); - NormalizedLandmarkList input_1 = - GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1); - NormalizedLandmarkList input_2 = - GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2); - std::vector inputs = {input_0, input_1, input_2}; - AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const NormalizedLandmarkList& result = - outputs[0].Get(); - ValidateCombinedLandmarks(inputs, result); -} - -TEST(ConcatenateNormalizedLandmarkListCalculatorTest, - TwoInputsAtTwoTimestamps) { - CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - NormalizedLandmarkList input_0 = - GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0); - NormalizedLandmarkList input_1 = - GenerateLandmarks(/*landmarks_size=*/1, /*value_multiplier=*/1); - NormalizedLandmarkList input_2 = - GenerateLandmarks(/*landmarks_size=*/2, /*value_multiplier=*/2); - std::vector inputs = {input_0, input_1, input_2}; - { AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); } - { AddInputLandmarkLists(inputs, /*timestamp=*/2, &runner); } - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(2, outputs.size()); - { - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const NormalizedLandmarkList& result = - outputs[0].Get(); - ValidateCombinedLandmarks(inputs, result); - } - { - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - const NormalizedLandmarkList& result = - outputs[1].Get(); - ValidateCombinedLandmarks(inputs, result); - } -} - -TEST(ConcatenateNormalizedLandmarkListCalculatorTest, - OneEmptyStreamStillOutput) { - CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator", - /*options_string=*/"", /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - NormalizedLandmarkList input_0 = - GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0); - std::vector inputs = {input_0}; - AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const NormalizedLandmarkList& result = - outputs[0].Get(); - ValidateCombinedLandmarks(inputs, result); -} - -TEST(ConcatenateNormalizedLandmarkListCalculatorTest, OneEmptyStreamNoOutput) { - CalculatorRunner runner("ConcatenateNormalizedLandmarkListCalculator", - /*options_string=*/ - "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: " - "{only_emit_if_all_present: true}", - /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - NormalizedLandmarkList input_0 = - GenerateLandmarks(/*landmarks_size=*/3, /*value_multiplier=*/0); - std::vector inputs = {input_0}; - AddInputLandmarkLists(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(0, outputs.size()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.cc b/mediapipe/calculators/core/concatenate_vector_calculator.cc deleted file mode 100644 index 20d6a3286..000000000 --- a/mediapipe/calculators/core/concatenate_vector_calculator.cc +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/concatenate_vector_calculator.h" - -#include - -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/util/render_data.pb.h" -#include "tensorflow/lite/interpreter.h" - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - -namespace mediapipe { - -// Example config: -// node { -// calculator: "ConcatenateFloatVectorCalculator" -// input_stream: "float_vector_1" -// input_stream: "float_vector_2" -// output_stream: "concatenated_float_vector" -// } -typedef ConcatenateVectorCalculator ConcatenateFloatVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateFloatVectorCalculator); - -// Example config: -// node { -// calculator: "ConcatenateInt32VectorCalculator" -// input_stream: "int32_vector_1" -// input_stream: "int32_vector_2" -// output_stream: "concatenated_int32_vector" -// } -typedef ConcatenateVectorCalculator ConcatenateInt32VectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateInt32VectorCalculator); - -typedef ConcatenateVectorCalculator ConcatenateUInt64VectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator); - -typedef ConcatenateVectorCalculator ConcatenateBoolVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator); - -// Example config: -// node { -// calculator: "ConcatenateTfLiteTensorVectorCalculator" -// input_stream: "tflitetensor_vector_1" -// input_stream: "tflitetensor_vector_2" -// output_stream: "concatenated_tflitetensor_vector" -// } -typedef ConcatenateVectorCalculator - ConcatenateTfLiteTensorVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateTfLiteTensorVectorCalculator); - -typedef ConcatenateVectorCalculator ConcatenateTensorVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateTensorVectorCalculator); - -typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark> - ConcatenateLandmarkVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarkVectorCalculator); - -typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmarkList> - ConcatenateLandmarListVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateLandmarListVectorCalculator); - -typedef ConcatenateVectorCalculator - ConcatenateClassificationListVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListVectorCalculator); - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer> - ConcatenateGlBufferVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateGlBufferVectorCalculator); -#endif - -typedef ConcatenateVectorCalculator - ConcatenateRenderDataVectorCalculator; -MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.h b/mediapipe/calculators/core/concatenate_vector_calculator.h deleted file mode 100644 index c6687814c..000000000 --- a/mediapipe/calculators/core/concatenate_vector_calculator.h +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_ - -#include -#include -#include - -#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/api2/port.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -// Note: since this is a calculator template that can be included by other -// source files, we do not place this in namespace api2 directly, but qualify -// the api2 names below, to avoid changing the visible name of the class. -// We cannot simply write "using mediapipe::api2" since it's a header file. -// This distinction will go away once api2 is finalized. - -// Concatenates several objects of type T or std::vector following stream -// index order. This class assumes that every input stream contains either T or -// vector type. To use this class for a particular type T, regisiter a -// calculator using ConcatenateVectorCalculator. -template -class ConcatenateVectorCalculator : public api2::Node { - public: - static constexpr - typename api2::Input>>::Multiple kIn{""}; - static constexpr api2::Output> kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK_GE(kIn(cc).Count(), 1); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - only_emit_if_all_present_ = - cc->Options<::mediapipe::ConcatenateVectorCalculatorOptions>() - .only_emit_if_all_present(); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (only_emit_if_all_present_) { - for (const auto& input : kIn(cc)) { - if (input.IsEmpty()) return ::absl::OkStatus(); - } - } - return ConcatenateVectors(std::is_copy_constructible(), cc); - } - - template - absl::Status ConcatenateVectors(std::true_type, CalculatorContext* cc) { - auto output = std::vector(); - for (const auto& input : kIn(cc)) { - if (input.IsEmpty()) continue; - input.Visit([&output](const U& value) { output.push_back(value); }, - [&output](const std::vector& value) { - output.insert(output.end(), value.begin(), value.end()); - }); - } - kOut(cc).Send(std::move(output)); - return absl::OkStatus(); - } - - template - absl::Status ConcatenateVectors(std::false_type, CalculatorContext* cc) { - return ConsumeAndConcatenateVectors(std::is_move_constructible(), cc); - } - - template - absl::Status ConsumeAndConcatenateVectors(std::true_type, - CalculatorContext* cc) { - auto output = std::vector(); - for (auto input : kIn(cc)) { - if (input.IsEmpty()) continue; - MP_RETURN_IF_ERROR(input.ConsumeAndVisit( - [&output](std::unique_ptr value) { - output.push_back(std::move(*value)); - }, - [&output](std::unique_ptr> value) { - output.insert(output.end(), std::make_move_iterator(value->begin()), - std::make_move_iterator(value->end())); - })); - } - kOut(cc).Send(std::move(output)); - return absl::OkStatus(); - } - - template - absl::Status ConsumeAndConcatenateVectors(std::false_type, - CalculatorContext* cc) { - return absl::InternalError( - "Cannot copy or move inputs to concatenate them"); - } - - private: - bool only_emit_if_all_present_; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_CORE_CONCATENATE_VECTOR_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.proto b/mediapipe/calculators/core/concatenate_vector_calculator.proto deleted file mode 100644 index 3753ffb5d..000000000 --- a/mediapipe/calculators/core/concatenate_vector_calculator.proto +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message ConcatenateVectorCalculatorOptions { - extend CalculatorOptions { - optional ConcatenateVectorCalculatorOptions ext = 259397839; - } - - // If true, the calculator will only emit a packet at the given timestamp if - // all input streams have a non-empty packet (AND operation on streams). - optional bool only_emit_if_all_present = 1 [default = false]; -} diff --git a/mediapipe/calculators/core/concatenate_vector_calculator_test.cc b/mediapipe/calculators/core/concatenate_vector_calculator_test.cc deleted file mode 100644 index b71dadf04..000000000 --- a/mediapipe/calculators/core/concatenate_vector_calculator_test.cc +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/concatenate_vector_calculator.h" - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -typedef ConcatenateVectorCalculator TestConcatenateIntVectorCalculator; -MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator); - -void AddInputVector(int index, const std::vector& input, int64 timestamp, - CalculatorRunner* runner) { - runner->MutableInputs()->Index(index).packets.push_back( - MakePacket>(input).At(Timestamp(timestamp))); -} - -void AddInputVectors(const std::vector>& inputs, - int64 timestamp, CalculatorRunner* runner) { - for (int i = 0; i < inputs.size(); ++i) { - AddInputVector(i, inputs[i], timestamp, runner); - } -} - -void AddInputItem(int index, int input, int64 timestamp, - CalculatorRunner* runner) { - runner->MutableInputs()->Index(index).packets.push_back( - MakePacket(input).At(Timestamp(timestamp))); -} - -void AddInputItems(const std::vector& inputs, int64 timestamp, - CalculatorRunner* runner) { - for (int i = 0; i < inputs.size(); ++i) { - AddInputItem(i, inputs[i], timestamp, runner); - } -} - -TEST(TestConcatenateIntVectorCalculatorTest, EmptyVectorInputs) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{}, {}, {}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_TRUE(outputs[0].Get>().empty()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, OneTimestamp) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{1, 2, 3}, {4}, {5, 6}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3, 4, 5, 6}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, TwoInputsAtTwoTimestamps) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - { - std::vector> inputs = {{1, 2, 3}, {4}, {5, 6}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - } - { - std::vector> inputs = {{0, 2}, {1}, {3, 5}}; - AddInputVectors(inputs, /*timestamp=*/2, &runner); - } - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(2, outputs.size()); - { - EXPECT_EQ(6, outputs[0].Get>().size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3, 4, 5, 6}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); - } - { - EXPECT_EQ(5, outputs[1].Get>().size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - std::vector expected_vector = {0, 2, 1, 3, 5}; - EXPECT_EQ(expected_vector, outputs[1].Get>()); - } -} - -TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamStillOutput) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{1, 2, 3}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, OneEmptyStreamNoOutput) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/ - "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: " - "{only_emit_if_all_present: true}", - /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{1, 2, 3}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(0, outputs.size()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneTimestamp) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector inputs = {1, 2, 3}; - AddInputItems(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, ItemsTwoInputsAtTwoTimestamps) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - { - std::vector inputs = {1, 2, 3}; - AddInputItems(inputs, /*timestamp=*/1, &runner); - } - { - std::vector inputs = {4, 5, 6}; - AddInputItems(inputs, /*timestamp=*/2, &runner); - } - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(2, outputs.size()); - { - EXPECT_EQ(3, outputs[0].Get>().size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); - } - { - EXPECT_EQ(3, outputs[1].Get>().size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - std::vector expected_vector = {4, 5, 6}; - EXPECT_EQ(expected_vector, outputs[1].Get>()); - } -} - -TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamStillOutput) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - // No third input item. - std::vector inputs = {1, 2}; - AddInputItems(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, ItemsOneEmptyStreamNoOutput) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/ - "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: " - "{only_emit_if_all_present: true}", - /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - // No third input item. - std::vector inputs = {1, 2}; - AddInputItems(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(0, outputs.size()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItems) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/4, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector vector_0 = {1, 2}; - std::vector vector_1 = {3, 4, 5}; - int item_0 = 6; - int item_1 = 7; - - AddInputVector(/*index*/ 0, vector_0, /*timestamp=*/1, &runner); - AddInputVector(/*index*/ 1, vector_1, /*timestamp=*/1, &runner); - AddInputItem(/*index*/ 2, item_0, /*timestamp=*/1, &runner); - AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner); - - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3, 4, 5, 6, 7}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(TestConcatenateIntVectorCalculatorTest, MixedVectorsAndItemsAnother) { - CalculatorRunner runner("TestConcatenateIntVectorCalculator", - /*options_string=*/"", /*num_inputs=*/4, - /*num_outputs=*/1, /*num_side_packets=*/0); - - int item_0 = 1; - std::vector vector_0 = {2, 3}; - std::vector vector_1 = {4, 5, 6}; - int item_1 = 7; - - AddInputItem(/*index*/ 0, item_0, /*timestamp=*/1, &runner); - AddInputVector(/*index*/ 1, vector_0, /*timestamp=*/1, &runner); - AddInputVector(/*index*/ 2, vector_1, /*timestamp=*/1, &runner); - AddInputItem(/*index*/ 3, item_1, /*timestamp=*/1, &runner); - - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1, 2, 3, 4, 5, 6, 7}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -void AddInputVectors(const std::vector>& inputs, - int64 timestamp, CalculatorRunner* runner) { - for (int i = 0; i < inputs.size(); ++i) { - runner->MutableInputs()->Index(i).packets.push_back( - MakePacket>(inputs[i]).At(Timestamp(timestamp))); - } -} - -TEST(ConcatenateFloatVectorCalculatorTest, EmptyVectorInputs) { - CalculatorRunner runner("ConcatenateFloatVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{}, {}, {}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_TRUE(outputs[0].Get>().empty()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); -} - -TEST(ConcatenateFloatVectorCalculatorTest, OneTimestamp) { - CalculatorRunner runner("ConcatenateFloatVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = { - {1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(ConcatenateFloatVectorCalculatorTest, TwoInputsAtTwoTimestamps) { - CalculatorRunner runner("ConcatenateFloatVectorCalculator", - /*options_string=*/"", /*num_inputs=*/3, - /*num_outputs=*/1, /*num_side_packets=*/0); - { - std::vector> inputs = { - {1.0f, 2.0f, 3.0f}, {4.0f}, {5.0f, 6.0f}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - } - { - std::vector> inputs = { - {0.0f, 2.0f}, {1.0f}, {3.0f, 5.0f}}; - AddInputVectors(inputs, /*timestamp=*/2, &runner); - } - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(2, outputs.size()); - { - EXPECT_EQ(6, outputs[0].Get>().size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); - } - { - EXPECT_EQ(5, outputs[1].Get>().size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - std::vector expected_vector = {0.0f, 2.0f, 1.0f, 3.0f, 5.0f}; - EXPECT_EQ(expected_vector, outputs[1].Get>()); - } -} - -TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamStillOutput) { - CalculatorRunner runner("ConcatenateFloatVectorCalculator", - /*options_string=*/"", /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{1.0f, 2.0f, 3.0f}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - std::vector expected_vector = {1.0f, 2.0f, 3.0f}; - EXPECT_EQ(expected_vector, outputs[0].Get>()); -} - -TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) { - CalculatorRunner runner("ConcatenateFloatVectorCalculator", - /*options_string=*/ - "[mediapipe.ConcatenateVectorCalculatorOptions.ext]: " - "{only_emit_if_all_present: true}", - /*num_inputs=*/2, - /*num_outputs=*/1, /*num_side_packets=*/0); - - std::vector> inputs = {{1.0f, 2.0f, 3.0f}}; - AddInputVectors(inputs, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - const std::vector& outputs = runner.Outputs().Index(0).packets; - EXPECT_EQ(0, outputs.size()); -} - -typedef ConcatenateVectorCalculator> - TestConcatenateUniqueIntPtrCalculator; -MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator); - -TEST(TestConcatenateUniqueIntVectorCalculatorTest, ConsumeOneTimestamp) { - /* Note: We don't use CalculatorRunner for this test because it keeps copies - * of input packets, so packets sent to the graph don't have sole ownership. - * The test needs to send packets that own the data. - */ - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: "in_1" - input_stream: "in_2" - input_stream: "in_3" - node { - calculator: "TestConcatenateUniqueIntPtrCalculator" - input_stream: "in_1" - input_stream: "in_2" - input_stream: "in_3" - output_stream: "out" - } - )pb"); - - std::vector outputs; - tool::AddVectorSink("out", &graph_config, &outputs); - - CalculatorGraph graph; - MP_EXPECT_OK(graph.Initialize(graph_config)); - MP_EXPECT_OK(graph.StartRun({})); - - // input1 : {0, 1, 2} - std::unique_ptr>> input_1 = - absl::make_unique>>(3); - for (int i = 0; i < 3; ++i) { - input_1->at(i) = absl::make_unique(i); - } - // input2: {3} - std::unique_ptr>> input_2 = - absl::make_unique>>(1); - input_2->at(0) = absl::make_unique(3); - // input3: {4, 5} - std::unique_ptr>> input_3 = - absl::make_unique>>(2); - input_3->at(0) = absl::make_unique(4); - input_3->at(1) = absl::make_unique(5); - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in_1", Adopt(input_1.release()).At(Timestamp(1)))); - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in_2", Adopt(input_2.release()).At(Timestamp(1)))); - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in_3", Adopt(input_3.release()).At(Timestamp(1)))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.CloseAllPacketSources()); - MP_EXPECT_OK(graph.WaitUntilDone()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector>& result = - outputs[0].Get>>(); - EXPECT_EQ(6, result.size()); - for (int i = 0; i < 6; ++i) { - const std::unique_ptr& v = result[i]; - EXPECT_EQ(i, *v); - } -} - -TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamStillOutput) { - /* Note: We don't use CalculatorRunner for this test because it keeps copies - * of input packets, so packets sent to the graph don't have sole ownership. - * The test needs to send packets that own the data. - */ - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: "in_1" - input_stream: "in_2" - node { - calculator: "TestConcatenateUniqueIntPtrCalculator" - input_stream: "in_1" - input_stream: "in_2" - output_stream: "out" - } - )pb"); - - std::vector outputs; - tool::AddVectorSink("out", &graph_config, &outputs); - - CalculatorGraph graph; - MP_EXPECT_OK(graph.Initialize(graph_config)); - MP_EXPECT_OK(graph.StartRun({})); - - // input1 : {0, 1, 2} - std::unique_ptr>> input_1 = - absl::make_unique>>(3); - for (int i = 0; i < 3; ++i) { - input_1->at(i) = absl::make_unique(i); - } - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in_1", Adopt(input_1.release()).At(Timestamp(1)))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.CloseAllPacketSources()); - MP_EXPECT_OK(graph.WaitUntilDone()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - const std::vector>& result = - outputs[0].Get>>(); - EXPECT_EQ(3, result.size()); - for (int i = 0; i < 3; ++i) { - const std::unique_ptr& v = result[i]; - EXPECT_EQ(i, *v); - } -} - -TEST(TestConcatenateUniqueIntVectorCalculatorTest, OneEmptyStreamNoOutput) { - /* Note: We don't use CalculatorRunner for this test because it keeps copies - * of input packets, so packets sent to the graph don't have sole ownership. - * The test needs to send packets that own the data. - */ - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: "in_1" - input_stream: "in_2" - node { - calculator: "TestConcatenateUniqueIntPtrCalculator" - input_stream: "in_1" - input_stream: "in_2" - output_stream: "out" - options { - [mediapipe.ConcatenateVectorCalculatorOptions.ext] { - only_emit_if_all_present: true - } - } - } - )pb"); - - std::vector outputs; - tool::AddVectorSink("out", &graph_config, &outputs); - - CalculatorGraph graph; - MP_EXPECT_OK(graph.Initialize(graph_config)); - MP_EXPECT_OK(graph.StartRun({})); - - // input1 : {0, 1, 2} - std::unique_ptr>> input_1 = - absl::make_unique>>(3); - for (int i = 0; i < 3; ++i) { - input_1->at(i) = absl::make_unique(i); - } - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in_1", Adopt(input_1.release()).At(Timestamp(1)))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.CloseAllPacketSources()); - MP_EXPECT_OK(graph.WaitUntilDone()); - - EXPECT_EQ(0, outputs.size()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.cc b/mediapipe/calculators/core/constant_side_packet_calculator.cc deleted file mode 100644 index ff328377e..000000000 --- a/mediapipe/calculators/core/constant_side_packet_calculator.cc +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/constant_side_packet_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace {} // namespace - -// Generates an output side packet or multiple output side packets according to -// the specified options. -// -// Example configs: -// node { -// calculator: "ConstantSidePacketCalculator" -// output_side_packet: "PACKET:packet" -// options: { -// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { -// packet { int_value: 2 } -// } -// } -// } -// -// node { -// calculator: "ConstantSidePacketCalculator" -// output_side_packet: "PACKET:0:int_packet" -// output_side_packet: "PACKET:1:bool_packet" -// options: { -// [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { -// packet { int_value: 2 } -// packet { bool_value: true } -// } -// } -// } -class ConstantSidePacketCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = - cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>(); - RET_CHECK_EQ(cc->OutputSidePackets().NumEntries(kPacketTag), - options.packet_size()) - << "Number of output side packets has to be same as number of packets " - "configured in options."; - - int index = 0; - for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag); - id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) { - const auto& packet_options = options.packet(index); - auto& packet = cc->OutputSidePackets().Get(id); - if (packet_options.has_int_value()) { - packet.Set(); - } else if (packet_options.has_float_value()) { - packet.Set(); - } else if (packet_options.has_bool_value()) { - packet.Set(); - } else if (packet_options.has_string_value()) { - packet.Set(); - } else if (packet_options.has_uint64_value()) { - packet.Set(); - } else if (packet_options.has_classification_list_value()) { - packet.Set(); - } else { - return absl::InvalidArgumentError( - "None of supported values were specified in options."); - } - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const auto& options = - cc->Options<::mediapipe::ConstantSidePacketCalculatorOptions>(); - int index = 0; - for (CollectionItemId id = cc->OutputSidePackets().BeginId(kPacketTag); - id != cc->OutputSidePackets().EndId(kPacketTag); ++id, ++index) { - auto& packet = cc->OutputSidePackets().Get(id); - const auto& packet_options = options.packet(index); - if (packet_options.has_int_value()) { - packet.Set(MakePacket(packet_options.int_value())); - } else if (packet_options.has_float_value()) { - packet.Set(MakePacket(packet_options.float_value())); - } else if (packet_options.has_bool_value()) { - packet.Set(MakePacket(packet_options.bool_value())); - } else if (packet_options.has_string_value()) { - packet.Set(MakePacket(packet_options.string_value())); - } else if (packet_options.has_uint64_value()) { - packet.Set(MakePacket(packet_options.uint64_value())); - } else if (packet_options.has_classification_list_value()) { - packet.Set(MakePacket( - packet_options.classification_list_value())); - } else { - return absl::InvalidArgumentError( - "None of supported values were specified in options."); - } - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } - - private: - static constexpr const char* kPacketTag = "PACKET"; -}; - -REGISTER_CALCULATOR(ConstantSidePacketCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.proto b/mediapipe/calculators/core/constant_side_packet_calculator.proto deleted file mode 100644 index 57f5dc545..000000000 --- a/mediapipe/calculators/core/constant_side_packet_calculator.proto +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/framework/formats/classification.proto"; - -option objc_class_prefix = "MediaPipe"; - -message ConstantSidePacketCalculatorOptions { - extend CalculatorOptions { - optional ConstantSidePacketCalculatorOptions ext = 291214597; - } - - message ConstantSidePacket { - oneof value { - int32 int_value = 1; - float float_value = 2; - bool bool_value = 3; - string string_value = 4; - uint64 uint64_value = 5; - ClassificationList classification_list_value = 6; - } - } - - repeated ConstantSidePacket packet = 1; -} diff --git a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc deleted file mode 100644 index a7ff808f4..000000000 --- a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -template -void DoTestSingleSidePacket(absl::string_view packet_spec, - const T& expected_value) { - static constexpr absl::string_view graph_config_template = R"( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:packet" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet $0 - } - } - } - )"; - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(graph_config_template, packet_spec)); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - MP_ASSERT_OK(graph.GetOutputSidePacket("packet")); - auto actual_value = - graph.GetOutputSidePacket("packet").value().template Get(); - EXPECT_EQ(actual_value, expected_value); -} - -TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) { - DoTestSingleSidePacket("{ int_value: 2 }", 2); - DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f); - DoTestSingleSidePacket("{ bool_value: true }", true); - DoTestSingleSidePacket(R"({ string_value: "str" })", "str"); -} - -TEST(ConstantSidePacketCalculatorTest, MultiplePackets) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:int_packet" - output_side_packet: "PACKET:1:float_packet" - output_side_packet: "PACKET:2:bool_packet" - output_side_packet: "PACKET:3:string_packet" - output_side_packet: "PACKET:4:another_string_packet" - output_side_packet: "PACKET:5:another_int_packet" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { int_value: 256 } - packet { float_value: 0.5f } - packet { bool_value: false } - packet { string_value: "string" } - packet { string_value: "another string" } - packet { int_value: 128 } - } - } - } - )pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("int_packet").value().Get(), 256); - MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("float_packet").value().Get(), - 0.5f); - MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet")); - EXPECT_FALSE(graph.GetOutputSidePacket("bool_packet").value().Get()); - MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet")); - EXPECT_EQ( - graph.GetOutputSidePacket("string_packet").value().Get(), - "string"); - MP_ASSERT_OK(graph.GetOutputSidePacket("another_string_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("another_string_packet") - .value() - .Get(), - "another string"); - MP_ASSERT_OK(graph.GetOutputSidePacket("another_int_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("another_int_packet").value().Get(), - 128); -} - -TEST(ConstantSidePacketCalculatorTest, ProcessingPacketsWithCorrectTagOnly) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:int_packet" - output_side_packet: "no_tag0" - output_side_packet: "PACKET:1:float_packet" - output_side_packet: "INCORRECT_TAG:0:name1" - output_side_packet: "PACKET:2:bool_packet" - output_side_packet: "PACKET:3:string_packet" - output_side_packet: "no_tag2" - output_side_packet: "INCORRECT_TAG:1:name2" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { int_value: 256 } - packet { float_value: 0.5f } - packet { bool_value: false } - packet { string_value: "string" } - } - } - } - )pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - MP_ASSERT_OK(graph.GetOutputSidePacket("int_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("int_packet").value().Get(), 256); - MP_ASSERT_OK(graph.GetOutputSidePacket("float_packet")); - EXPECT_EQ(graph.GetOutputSidePacket("float_packet").value().Get(), - 0.5f); - MP_ASSERT_OK(graph.GetOutputSidePacket("bool_packet")); - EXPECT_FALSE(graph.GetOutputSidePacket("bool_packet").value().Get()); - MP_ASSERT_OK(graph.GetOutputSidePacket("string_packet")); - EXPECT_EQ( - graph.GetOutputSidePacket("string_packet").value().Get(), - "string"); -} - -TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MoreOptionsThanPackets) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:int_packet" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { int_value: 256 } - packet { float_value: 0.5f } - } - } - } - )pb"); - CalculatorGraph graph; - EXPECT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST(ConstantSidePacketCalculatorTest, IncorrectConfig_MorePacketsThanOptions) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:int_packet" - output_side_packet: "PACKET:1:float_packet" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { int_value: 256 } - } - } - } - )pb"); - CalculatorGraph graph; - EXPECT_FALSE(graph.Initialize(graph_config).ok()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/counting_source_calculator.cc b/mediapipe/calculators/core/counting_source_calculator.cc deleted file mode 100644 index 0b731d9ce..000000000 --- a/mediapipe/calculators/core/counting_source_calculator.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -// Source calculator that produces MAX_COUNT*BATCH_SIZE int packets of -// sequential numbers from INITIAL_VALUE (default 0) with a common -// difference of INCREMENT (default 1) between successive numbers (with -// timestamps corresponding to the sequence numbers). The packets are -// produced in BATCH_SIZE sized batches with each call to Process(). An -// error will be returned after ERROR_COUNT batches. An error will be -// produced in Open() if ERROR_ON_OPEN is true. Either MAX_COUNT or -// ERROR_COUNT must be provided and non-negative. If BATCH_SIZE is not -// provided, then batches are of size 1. -class CountingSourceCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Outputs().Index(0).Set(); - - if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN")) { - cc->InputSidePackets().Tag("ERROR_ON_OPEN").Set(); - } - - RET_CHECK(cc->InputSidePackets().HasTag("MAX_COUNT") || - cc->InputSidePackets().HasTag("ERROR_COUNT")); - if (cc->InputSidePackets().HasTag("MAX_COUNT")) { - cc->InputSidePackets().Tag("MAX_COUNT").Set(); - } - if (cc->InputSidePackets().HasTag("ERROR_COUNT")) { - cc->InputSidePackets().Tag("ERROR_COUNT").Set(); - } - - if (cc->InputSidePackets().HasTag("BATCH_SIZE")) { - cc->InputSidePackets().Tag("BATCH_SIZE").Set(); - } - if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) { - cc->InputSidePackets().Tag("INITIAL_VALUE").Set(); - } - if (cc->InputSidePackets().HasTag("INCREMENT")) { - cc->InputSidePackets().Tag("INCREMENT").Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - if (cc->InputSidePackets().HasTag("ERROR_ON_OPEN") && - cc->InputSidePackets().Tag("ERROR_ON_OPEN").Get()) { - return absl::NotFoundError("expected error"); - } - if (cc->InputSidePackets().HasTag("ERROR_COUNT")) { - error_count_ = cc->InputSidePackets().Tag("ERROR_COUNT").Get(); - RET_CHECK_LE(0, error_count_); - } - if (cc->InputSidePackets().HasTag("MAX_COUNT")) { - max_count_ = cc->InputSidePackets().Tag("MAX_COUNT").Get(); - RET_CHECK_LE(0, max_count_); - } - if (cc->InputSidePackets().HasTag("BATCH_SIZE")) { - batch_size_ = cc->InputSidePackets().Tag("BATCH_SIZE").Get(); - RET_CHECK_LT(0, batch_size_); - } - if (cc->InputSidePackets().HasTag("INITIAL_VALUE")) { - counter_ = cc->InputSidePackets().Tag("INITIAL_VALUE").Get(); - } - if (cc->InputSidePackets().HasTag("INCREMENT")) { - increment_ = cc->InputSidePackets().Tag("INCREMENT").Get(); - RET_CHECK_LT(0, increment_); - } - RET_CHECK(error_count_ >= 0 || max_count_ >= 0); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (error_count_ >= 0 && batch_counter_ >= error_count_) { - return absl::InternalError("expected error"); - } - if (max_count_ >= 0 && batch_counter_ >= max_count_) { - return tool::StatusStop(); - } - for (int i = 0; i < batch_size_; ++i) { - cc->Outputs().Index(0).Add(new int(counter_), Timestamp(counter_)); - counter_ += increment_; - } - ++batch_counter_; - return absl::OkStatus(); - } - - private: - int max_count_ = -1; - int error_count_ = -1; - int batch_size_ = 1; - int batch_counter_ = 0; - int counter_ = 0; - int increment_ = 1; -}; -REGISTER_CALCULATOR(CountingSourceCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/default_side_packet_calculator.cc b/mediapipe/calculators/core/default_side_packet_calculator.cc deleted file mode 100644 index 145d06389..000000000 --- a/mediapipe/calculators/core/default_side_packet_calculator.cc +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace { - -constexpr char kOptionalValueTag[] = "OPTIONAL_VALUE"; -constexpr char kDefaultValueTag[] = "DEFAULT_VALUE"; -constexpr char kValueTag[] = "VALUE"; - -} // namespace - -// Outputs side packet default value if optional value is not provided. -// -// This calculator utilizes the fact that MediaPipe automatically removes -// optional side packets of the calculator configuration (i.e. OPTIONAL_VALUE). -// And if it happens - returns default value, otherwise - returns optional -// value. -// -// Input: -// OPTIONAL_VALUE (optional) - AnyType (but same type as DEFAULT_VALUE) -// Optional side packet value that is outputted by the calculator as is if -// provided. -// -// DEFAULT_VALUE - AnyType -// Default side pack value that is outputted by the calculator if -// OPTIONAL_VALUE is not provided. -// -// Output: -// VALUE - AnyType (but same type as DEFAULT_VALUE) -// Either OPTIONAL_VALUE (if provided) or DEFAULT_VALUE (otherwise). -// -// Usage example: -// node { -// calculator: "DefaultSidePacketCalculator" -// input_side_packet: "OPTIONAL_VALUE:segmentation_mask_enabled_optional" -// input_side_packet: "DEFAULT_VALUE:segmentation_mask_enabled_default" -// output_side_packet: "VALUE:segmentation_mask_enabled" -// } -class DefaultSidePacketCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(DefaultSidePacketCalculator); - -absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->InputSidePackets().HasTag(kDefaultValueTag)) - << "Default value must be provided"; - cc->InputSidePackets().Tag(kDefaultValueTag).SetAny(); - - // Optional input side packet can be unspecified. In this case MediaPipe will - // remove it from the calculator config. - if (cc->InputSidePackets().HasTag(kOptionalValueTag)) { - cc->InputSidePackets() - .Tag(kOptionalValueTag) - .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag)) - .Optional(); - } - - RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag)); - cc->OutputSidePackets().Tag(kValueTag).SetSameAs( - &cc->InputSidePackets().Tag(kDefaultValueTag)); - - return absl::OkStatus(); -} - -absl::Status DefaultSidePacketCalculator::Open(CalculatorContext* cc) { - // If optional value is provided it is returned as the calculator output. - if (cc->InputSidePackets().HasTag(kOptionalValueTag)) { - auto& packet = cc->InputSidePackets().Tag(kOptionalValueTag); - cc->OutputSidePackets().Tag(kValueTag).Set(packet); - return absl::OkStatus(); - } - - // If no optional value - auto& packet = cc->InputSidePackets().Tag(kDefaultValueTag); - cc->OutputSidePackets().Tag(kValueTag).Set(packet); - - return absl::OkStatus(); -} - -absl::Status DefaultSidePacketCalculator::Process(CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/dequantize_byte_array_calculator.cc b/mediapipe/calculators/core/dequantize_byte_array_calculator.cc deleted file mode 100644 index 04a7e55a0..000000000 --- a/mediapipe/calculators/core/dequantize_byte_array_calculator.cc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/dequantize_byte_array_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/status.h" - -// Dequantizes a byte array to a vector of floats. -// -// Example config: -// node { -// calculator: "DequantizeByteArrayCalculator" -// input_stream: "ENCODED:encoded" -// output_stream: "FLOAT_VECTOR:float_vector" -// options { -// [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: { -// max_quantized_value: 2 -// min_quantized_value: -2 -// } -// } -// } -namespace mediapipe { - -class DequantizeByteArrayCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag("ENCODED").Set(); - cc->Outputs().Tag("FLOAT_VECTOR").Set>(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - const auto options = - cc->Options<::mediapipe::DequantizeByteArrayCalculatorOptions>(); - if (!options.has_max_quantized_value() || - !options.has_min_quantized_value()) { - return absl::InvalidArgumentError( - "Both max_quantized_value and min_quantized_value must be provided " - "in DequantizeByteArrayCalculatorOptions."); - } - float max_quantized_value = options.max_quantized_value(); - float min_quantized_value = options.min_quantized_value(); - if (max_quantized_value < min_quantized_value + FLT_EPSILON) { - return absl::InvalidArgumentError( - "max_quantized_value must be greater than min_quantized_value."); - } - float range = max_quantized_value - min_quantized_value; - scalar_ = range / 255.0; - bias_ = (range / 512.0) + min_quantized_value; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - const std::string& encoded = - cc->Inputs().Tag("ENCODED").Value().Get(); - std::vector float_vector; - float_vector.reserve(encoded.length()); - for (int i = 0; i < encoded.length(); ++i) { - float_vector.push_back( - static_cast(encoded.at(i)) * scalar_ + bias_); - } - cc->Outputs() - .Tag("FLOAT_VECTOR") - .AddPacket(MakePacket>(float_vector) - .At(cc->InputTimestamp())); - return absl::OkStatus(); - } - - private: - float scalar_; - float bias_; -}; - -REGISTER_CALCULATOR(DequantizeByteArrayCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/dequantize_byte_array_calculator.proto b/mediapipe/calculators/core/dequantize_byte_array_calculator.proto deleted file mode 100644 index 3af8e11ef..000000000 --- a/mediapipe/calculators/core/dequantize_byte_array_calculator.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message DequantizeByteArrayCalculatorOptions { - extend CalculatorOptions { - optional DequantizeByteArrayCalculatorOptions ext = 272316343; - } - - optional float max_quantized_value = 1; - optional float min_quantized_value = 2; -} diff --git a/mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc b/mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc deleted file mode 100644 index cf0a8dc15..000000000 --- a/mediapipe/calculators/core/dequantize_byte_array_calculator_test.cc +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "DequantizeByteArrayCalculator" - input_stream: "ENCODED:encoded" - output_stream: "FLOAT_VECTOR:float_vector" - options { - [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: { - max_quantized_value: 2 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::string empty_string; - runner.MutableInputs()->Tag("ENCODED").packets.push_back( - MakePacket(empty_string).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "Both max_quantized_value and min_quantized_value must be provided")); -} - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "DequantizeByteArrayCalculator" - input_stream: "ENCODED:encoded" - output_stream: "FLOAT_VECTOR:float_vector" - options { - [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: { - max_quantized_value: -2 - min_quantized_value: 2 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::string empty_string; - runner.MutableInputs()->Tag("ENCODED").packets.push_back( - MakePacket(empty_string).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "max_quantized_value must be greater than min_quantized_value")); -} - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "DequantizeByteArrayCalculator" - input_stream: "ENCODED:encoded" - output_stream: "FLOAT_VECTOR:float_vector" - options { - [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: { - max_quantized_value: 1 - min_quantized_value: 1 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::string empty_string; - runner.MutableInputs()->Tag("ENCODED").packets.push_back( - MakePacket(empty_string).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "max_quantized_value must be greater than min_quantized_value")); -} - -TEST(DequantizeByteArrayCalculatorTest, TestDequantization) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "DequantizeByteArrayCalculator" - input_stream: "ENCODED:encoded" - output_stream: "FLOAT_VECTOR:float_vector" - options { - [mediapipe.DequantizeByteArrayCalculatorOptions.ext]: { - max_quantized_value: 2 - min_quantized_value: -2 - } - } - )pb"); - CalculatorRunner runner(node_config); - unsigned char input[4] = {0x7F, 0xFF, 0x00, 0x01}; - runner.MutableInputs()->Tag("ENCODED").packets.push_back( - MakePacket( - std::string(reinterpret_cast(input), 4)) - .At(Timestamp(0))); - auto status = runner.Run(); - MP_ASSERT_OK(runner.Run()); - const std::vector& outputs = - runner.Outputs().Tag("FLOAT_VECTOR").packets; - EXPECT_EQ(1, outputs.size()); - const std::vector& result = outputs[0].Get>(); - ASSERT_FALSE(result.empty()); - EXPECT_EQ(4, result.size()); - EXPECT_NEAR(0, result[0], 0.01); - EXPECT_NEAR(2, result[1], 0.01); - EXPECT_NEAR(-2, result[2], 0.01); - EXPECT_NEAR(-1.976, result[3], 0.01); - - EXPECT_EQ(Timestamp(0), outputs[0].Timestamp()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc deleted file mode 100644 index 2a366f992..000000000 --- a/mediapipe/calculators/core/end_loop_calculator.cc +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/end_loop_calculator.h" - -#include - -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/util/render_data.pb.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { - -typedef EndLoopCalculator> - EndLoopNormalizedRectCalculator; -REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator); - -typedef EndLoopCalculator> - EndLoopLandmarkListVectorCalculator; -REGISTER_CALCULATOR(EndLoopLandmarkListVectorCalculator); - -typedef EndLoopCalculator> - EndLoopNormalizedLandmarkListVectorCalculator; -REGISTER_CALCULATOR(EndLoopNormalizedLandmarkListVectorCalculator); - -typedef EndLoopCalculator> EndLoopBooleanCalculator; -REGISTER_CALCULATOR(EndLoopBooleanCalculator); - -typedef EndLoopCalculator> - EndLoopRenderDataCalculator; -REGISTER_CALCULATOR(EndLoopRenderDataCalculator); - -typedef EndLoopCalculator> - EndLoopClassificationListCalculator; -REGISTER_CALCULATOR(EndLoopClassificationListCalculator); - -typedef EndLoopCalculator> EndLoopTensorCalculator; -REGISTER_CALCULATOR(EndLoopTensorCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.h b/mediapipe/calculators/core/end_loop_calculator.h deleted file mode 100644 index e40301e81..000000000 --- a/mediapipe/calculators/core/end_loop_calculator.h +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ - -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_contract.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Calculator for completing the processing of loops on iterable collections -// inside a MediaPipe graph. The EndLoopCalculator collects all input packets -// from ITEM input_stream into a collection and upon receiving the flush signal -// from the "BATCH_END" tagged input stream, it emits the aggregated results -// at the original timestamp contained in the "BATCH_END" input stream. -// -// It is designed to be used like: -// -// node { -// calculator: "BeginLoopWithIterableCalculator" -// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts -// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts -// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts -// } -// -// node { -// calculator: "ElementToBlaConverterSubgraph" -// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts -// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts -// } -// -// node { -// calculator: "EndLoopWithOutputCalculator" -// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts -// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts -// output_stream: "OUTPUT:aggregated_result" # IterableU @ext_ts -// } -template -class EndLoopCalculator : public CalculatorBase { - using ItemT = typename IterableT::value_type; - - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("BATCH_END")) - << "Missing BATCH_END tagged input_stream."; - cc->Inputs().Tag("BATCH_END").Set(); - - RET_CHECK(cc->Inputs().HasTag("ITEM")); - cc->Inputs().Tag("ITEM").Set(); - - RET_CHECK(cc->Outputs().HasTag("ITERABLE")); - cc->Outputs().Tag("ITERABLE").Set(); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (!cc->Inputs().Tag("ITEM").IsEmpty()) { - if (!input_stream_collection_) { - input_stream_collection_.reset(new IterableT); - } - input_stream_collection_->push_back( - cc->Inputs().Tag("ITEM").template Get()); - } - - if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) { // flush signal - Timestamp loop_control_ts = - cc->Inputs().Tag("BATCH_END").template Get(); - if (input_stream_collection_) { - cc->Outputs() - .Tag("ITERABLE") - .Add(input_stream_collection_.release(), loop_control_ts); - } else { - // Since there is no collection, inform downstream calculators to not - // expect any packet by updating the timestamp bounds. - cc->Outputs() - .Tag("ITERABLE") - .SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1)); - } - } - return absl::OkStatus(); - } - - private: - std::unique_ptr input_stream_collection_; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/flow_limiter_calculator.cc b/mediapipe/calculators/core/flow_limiter_calculator.cc deleted file mode 100644 index eba621ce3..000000000 --- a/mediapipe/calculators/core/flow_limiter_calculator.cc +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/header_util.h" - -namespace mediapipe { - -// FlowLimiterCalculator is used to limit the number of frames in flight -// by dropping input frames when necessary. -// -// The input stream "FINISH" is used to signal the FlowLimiterCalculator -// when a frame is finished processing. Either a non-empty "FINISH" packet -// or a timestamp bound should be received for each processed frame. -// -// The combination of `max_in_flight: 1` and `max_in_queue: 1` generally gives -// best throughput/latency balance. Throughput is nearly optimal as the -// graph is never idle as there is always something in the queue. Latency is -// nearly optimal latency as the queue always stores the latest available frame. -// -// Increasing `max_in_flight` to 2 or more can yield the better throughput -// when the graph exhibits a high degree of pipeline parallelism. Decreasing -// `max_in_flight` to 0 can yield a better average latency, but at the cost of -// lower throughput (lower framerate) due to the time during which the graph -// is idle awaiting the next input frame. -// -// Example config: -// node { -// calculator: "FlowLimiterCalculator" -// input_stream: "raw_frames" -// input_stream: "FINISHED:finished" -// input_stream_info: { -// tag_index: 'FINISHED' -// back_edge: true -// } -// output_stream: "sampled_frames" -// output_stream: "ALLOW:allowed_timestamps" -// } -// -// The "ALLOW" stream indicates the transition between accepting frames and -// dropping frames. "ALLOW = true" indicates the start of accepting frames -// including the current timestamp, and "ALLOW = false" indicates the start of -// dropping frames including the current timestamp. -// -// FlowLimiterCalculator provides limited support for multiple input streams. -// The first input stream is treated as the main input stream and successive -// input streams are treated as auxiliary input streams. The auxiliary input -// streams are limited to timestamps passed on the main input stream. -// -class FlowLimiterCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - auto& side_inputs = cc->InputSidePackets(); - side_inputs.Tag("OPTIONS").Set().Optional(); - cc->Inputs().Tag("OPTIONS").Set().Optional(); - RET_CHECK_GE(cc->Inputs().NumEntries(""), 1); - for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) { - cc->Inputs().Get("", i).SetAny(); - cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i))); - } - cc->Inputs().Get("FINISHED", 0).SetAny(); - cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set().Optional(); - cc->Outputs().Tag("ALLOW").Set().Optional(); - cc->SetInputStreamHandler("ImmediateInputStreamHandler"); - cc->SetProcessTimestampBounds(true); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - options_ = cc->Options(); - options_ = tool::RetrieveOptions(options_, cc->InputSidePackets()); - if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) { - options_.set_max_in_flight( - cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get()); - } - input_queues_.resize(cc->Inputs().NumEntries("")); - RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs()))); - return absl::OkStatus(); - } - - // Returns true if an additional frame can be released for processing. - // The "ALLOW" output stream indicates this condition at each input frame. - bool ProcessingAllowed() { - return frames_in_flight_.size() < options_.max_in_flight(); - } - - // Outputs a packet indicating whether a frame was sent or dropped. - void SendAllow(bool allow, Timestamp ts, CalculatorContext* cc) { - if (cc->Outputs().HasTag("ALLOW")) { - cc->Outputs().Tag("ALLOW").AddPacket(MakePacket(allow).At(ts)); - } - } - - // Sets the timestamp bound or closes an output stream. - void SetNextTimestampBound(Timestamp bound, OutputStream* stream) { - if (bound > Timestamp::Max()) { - stream->Close(); - } else { - stream->SetNextTimestampBound(bound); - } - } - - // Returns true if a certain timestamp is being processed. - bool IsInFlight(Timestamp timestamp) { - return std::find(frames_in_flight_.begin(), frames_in_flight_.end(), - timestamp) != frames_in_flight_.end(); - } - - // Releases input packets up to the latest settled input timestamp. - void ProcessAuxiliaryInputs(CalculatorContext* cc) { - Timestamp settled_bound = cc->Outputs().Get("", 0).NextTimestampBound(); - for (int i = 1; i < cc->Inputs().NumEntries(""); ++i) { - // Release settled frames from each input queue. - while (!input_queues_[i].empty() && - input_queues_[i].front().Timestamp() < settled_bound) { - Packet packet = input_queues_[i].front(); - input_queues_[i].pop_front(); - if (IsInFlight(packet.Timestamp())) { - cc->Outputs().Get("", i).AddPacket(packet); - } - } - - // Propagate each input timestamp bound. - if (!input_queues_[i].empty()) { - Timestamp bound = input_queues_[i].front().Timestamp(); - SetNextTimestampBound(bound, &cc->Outputs().Get("", i)); - } else { - Timestamp bound = - cc->Inputs().Get("", i).Value().Timestamp().NextAllowedInStream(); - SetNextTimestampBound(bound, &cc->Outputs().Get("", i)); - } - } - } - - // Releases input packets allowed by the max_in_flight constraint. - absl::Status Process(CalculatorContext* cc) final { - options_ = tool::RetrieveOptions(options_, cc->Inputs()); - - // Process the FINISHED input stream. - Packet finished_packet = cc->Inputs().Tag("FINISHED").Value(); - if (finished_packet.Timestamp() == cc->InputTimestamp()) { - while (!frames_in_flight_.empty() && - frames_in_flight_.front() <= finished_packet.Timestamp()) { - frames_in_flight_.pop_front(); - } - } - - // Process the frame input streams. - for (int i = 0; i < cc->Inputs().NumEntries(""); ++i) { - Packet packet = cc->Inputs().Get("", i).Value(); - if (!packet.IsEmpty()) { - input_queues_[i].push_back(packet); - } - } - - // Abandon expired frames in flight. Note that old frames are abandoned - // when much newer frame timestamps arrive regardless of elapsed time. - TimestampDiff timeout = options_.in_flight_timeout(); - Timestamp latest_ts = cc->Inputs().Get("", 0).Value().Timestamp(); - if (timeout > 0 && latest_ts == cc->InputTimestamp() && - latest_ts < Timestamp::Max()) { - while (!frames_in_flight_.empty() && - (latest_ts - frames_in_flight_.front()) > timeout) { - frames_in_flight_.pop_front(); - } - } - - // Release allowed frames from the main input queue. - auto& input_queue = input_queues_[0]; - while (ProcessingAllowed() && !input_queue.empty()) { - Packet packet = input_queue.front(); - input_queue.pop_front(); - cc->Outputs().Get("", 0).AddPacket(packet); - SendAllow(true, packet.Timestamp(), cc); - frames_in_flight_.push_back(packet.Timestamp()); - } - - // Limit the number of queued frames. - // Note that frames can be dropped after frames are released because - // frame-packets and FINISH-packets never arrive in the same Process call. - while (input_queue.size() > options_.max_in_queue()) { - Packet packet = input_queue.front(); - input_queue.pop_front(); - SendAllow(false, packet.Timestamp(), cc); - } - - // Propagate the input timestamp bound. - if (!input_queue.empty()) { - Timestamp bound = input_queue.front().Timestamp(); - SetNextTimestampBound(bound, &cc->Outputs().Get("", 0)); - } else { - Timestamp bound = - cc->Inputs().Get("", 0).Value().Timestamp().NextAllowedInStream(); - SetNextTimestampBound(bound, &cc->Outputs().Get("", 0)); - if (cc->Outputs().HasTag("ALLOW")) { - SetNextTimestampBound(bound, &cc->Outputs().Tag("ALLOW")); - } - } - - ProcessAuxiliaryInputs(cc); - return absl::OkStatus(); - } - - private: - FlowLimiterCalculatorOptions options_; - std::vector> input_queues_; - std::deque frames_in_flight_; -}; -REGISTER_CALCULATOR(FlowLimiterCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/flow_limiter_calculator.proto b/mediapipe/calculators/core/flow_limiter_calculator.proto deleted file mode 100644 index 0f7c925ae..000000000 --- a/mediapipe/calculators/core/flow_limiter_calculator.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message FlowLimiterCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional FlowLimiterCalculatorOptions ext = 326963320; - } - - // The maximum number of frames released for processing at one time. - // The default value limits to 1 frame processing at a time. - optional int32 max_in_flight = 1 [default = 1]; - - // The maximum number of frames queued waiting for processing. - // The default value limits to 1 frame awaiting processing. - optional int32 max_in_queue = 2 [default = 0]; - - // The maximum time in microseconds to wait for a frame to finish processing. - // The default value stops waiting after 1 sec. - // The value 0 specifies no timeout. - optional int64 in_flight_timeout = 3 [default = 1000000]; -} diff --git a/mediapipe/calculators/core/flow_limiter_calculator_test.cc b/mediapipe/calculators/core/flow_limiter_calculator_test.cc deleted file mode 100644 index d2294dd48..000000000 --- a/mediapipe/calculators/core/flow_limiter_calculator_test.cc +++ /dev/null @@ -1,760 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/time/clock.h" -#include "absl/time/time.h" -#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/simulation_clock.h" -#include "mediapipe/framework/tool/simulation_clock_executor.h" -#include "mediapipe/framework/tool/sink.h" - -namespace mediapipe { - -namespace { -// A simple Semaphore for synchronizing test threads. -class AtomicSemaphore { - public: - AtomicSemaphore(int64_t supply) : supply_(supply) {} - void Acquire(int64_t amount) { - while (supply_.fetch_sub(amount) - amount < 0) { - Release(amount); - } - } - void Release(int64_t amount) { supply_.fetch_add(amount); } - - private: - std::atomic supply_; -}; - -// Returns the timestamp values for a vector of Packets. -std::vector TimestampValues(const std::vector& packets) { - std::vector result; - for (const Packet& packet : packets) { - result.push_back(packet.Timestamp().Value()); - } - return result; -} - -// Returns the packet values for a vector of Packets. -template -std::vector PacketValues(const std::vector& packets) { - std::vector result; - for (const Packet& packet : packets) { - result.push_back(packet.Get()); - } - return result; -} - -// A Calculator::Process callback function. -typedef std::function - ProcessFunction; - -// A testing callback function that passes through all packets. -absl::Status PassthroughFunction(const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - for (int i = 0; i < inputs.NumEntries(); ++i) { - if (!inputs.Index(i).Value().IsEmpty()) { - outputs->Index(i).AddPacket(inputs.Index(i).Value()); - } - } - return absl::OkStatus(); -} - -// Tests demonstrating an FlowLimiterCalculator operating in a cyclic graph. -class FlowLimiterCalculatorSemaphoreTest : public testing::Test { - public: - FlowLimiterCalculatorSemaphoreTest() : exit_semaphore_(0) {} - - void SetUp() override { - graph_config_ = InflightGraphConfig(); - tool::AddVectorSink("out_1", &graph_config_, &out_1_packets_); - } - - void InitializeGraph(int max_in_flight) { - ProcessFunction semaphore_1_func = [&](const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - exit_semaphore_.Acquire(1); - return PassthroughFunction(inputs, outputs); - }; - FlowLimiterCalculatorOptions options; - options.set_max_in_flight(max_in_flight); - options.set_max_in_queue(1); - MP_ASSERT_OK(graph_.Initialize( - graph_config_, { - {"limiter_options", Adopt(new auto(options))}, - {"callback_1", Adopt(new auto(semaphore_1_func))}, - })); - - allow_poller_.reset( - new OutputStreamPoller(graph_.AddOutputStreamPoller("allow").value())); - } - - // Adds a packet to a graph input stream. - void AddPacket(const std::string& input_name, int value) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(value).At(Timestamp(value)))); - } - - // A calculator graph starting with an FlowLimiterCalculator and - // ending with a InFlightFinishCalculator. - // Back-edge "finished" limits processing to one frame in-flight. - // The LambdaCalculator is used to keep certain frames in flight. - CalculatorGraphConfig InflightGraphConfig() { - return ParseTextProtoOrDie(R"pb( - input_stream: 'in_1' - node { - calculator: 'FlowLimiterCalculator' - input_side_packet: 'OPTIONS:limiter_options' - input_stream: 'in_1' - input_stream: 'FINISHED:out_1' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_1_sampled' - output_stream: 'ALLOW:allow' - } - node { - calculator: 'LambdaCalculator' - input_side_packet: 'callback_1' - input_stream: 'in_1_sampled' - output_stream: 'out_1' - } - )pb"); - } - - protected: - CalculatorGraphConfig graph_config_; - CalculatorGraph graph_; - AtomicSemaphore exit_semaphore_; - std::vector out_1_packets_; - std::unique_ptr allow_poller_; -}; - -// A test demonstrating an FlowLimiterCalculator operating in a cyclic -// graph. This test shows that: -// -// (1) Frames exceeding the queue size are dropped. -// (2) The "ALLOW" signal is produced. -// (3) Timestamps are passed through unaltered. -// -TEST_F(FlowLimiterCalculatorSemaphoreTest, FramesDropped) { - InitializeGraph(1); - MP_ASSERT_OK(graph_.StartRun({})); - - auto send_packet = [this](const std::string& input_name, int64 n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - - Packet allow_packet; - send_packet("in_1", 0); - for (int i = 0; i < 9; i++) { - EXPECT_TRUE(allow_poller_->Next(&allow_packet)); - EXPECT_TRUE(allow_packet.Get()); - // This input should wait in the limiter input queue. - send_packet("in_1", i * 10 + 5); - // This input should drop the previous input. - send_packet("in_1", i * 10 + 10); - EXPECT_TRUE(allow_poller_->Next(&allow_packet)); - EXPECT_FALSE(allow_packet.Get()); - exit_semaphore_.Release(1); - } - exit_semaphore_.Release(1); - MP_EXPECT_OK(graph_.CloseInputStream("in_1")); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - - // All output streams are closed and all output packets are delivered, - // with stream "in_1" closed. - EXPECT_EQ(10, out_1_packets_.size()); - - // Timestamps have not been altered. - EXPECT_EQ(PacketValues(out_1_packets_), - TimestampValues(out_1_packets_)); - - // Extra inputs on in_1 have been dropped. - EXPECT_EQ(TimestampValues(out_1_packets_), - (std::vector{0, 10, 20, 30, 40, 50, 60, 70, 80, 90})); -} - -// A calculator that sleeps during Process. -class SleepCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag("PACKET").SetAny(); - cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET")); - cc->InputSidePackets().Tag("SLEEP_TIME").Set(); - cc->InputSidePackets().Tag("WARMUP_TIME").Set(); - cc->InputSidePackets().Tag("CLOCK").Set(); - cc->SetTimestampOffset(0); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - clock_ = cc->InputSidePackets().Tag("CLOCK").Get(); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - ++packet_count; - absl::Duration sleep_time = absl::Microseconds( - packet_count == 1 - ? cc->InputSidePackets().Tag("WARMUP_TIME").Get() - : cc->InputSidePackets().Tag("SLEEP_TIME").Get()); - clock_->Sleep(sleep_time); - cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value()); - return absl::OkStatus(); - } - - private: - ::mediapipe::Clock* clock_ = nullptr; - int packet_count = 0; -}; -REGISTER_CALCULATOR(SleepCalculator); - -// A calculator that drops a packet occasionally. -// Drops the 3rd packet, and optionally the corresponding timestamp bound. -class DropCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag("PACKET").SetAny(); - cc->Outputs().Tag("PACKET").SetSameAs(&cc->Inputs().Tag("PACKET")); - cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Set(); - cc->SetProcessTimestampBounds(true); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - if (!cc->Inputs().Tag("PACKET").Value().IsEmpty()) { - ++packet_count; - } - bool drop = (packet_count == 3); - if (!drop && !cc->Inputs().Tag("PACKET").Value().IsEmpty()) { - cc->Outputs().Tag("PACKET").AddPacket(cc->Inputs().Tag("PACKET").Value()); - } - if (!drop || !cc->InputSidePackets().Tag("DROP_TIMESTAMPS").Get()) { - cc->Outputs().Tag("PACKET").SetNextTimestampBound( - cc->InputTimestamp().NextAllowedInStream()); - } - return absl::OkStatus(); - } - - private: - int packet_count = 0; -}; -REGISTER_CALCULATOR(DropCalculator); - -// Tests demonstrating an FlowLimiterCalculator processing FINISHED timestamps. -class FlowLimiterCalculatorTest : public testing::Test { - protected: - CalculatorGraphConfig InflightGraphConfig() { - return ParseTextProtoOrDie(R"pb( - input_stream: 'in_1' - node { - calculator: 'FlowLimiterCalculator' - input_side_packet: 'OPTIONS:limiter_options' - input_stream: 'in_1' - input_stream: 'FINISHED:out_1' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_1_sampled' - output_stream: 'ALLOW:allow' - } - node { - calculator: 'SleepCalculator' - input_side_packet: 'WARMUP_TIME:warmup_time' - input_side_packet: 'SLEEP_TIME:sleep_time' - input_side_packet: 'CLOCK:clock' - input_stream: 'PACKET:in_1_sampled' - output_stream: 'PACKET:out_1_sampled' - } - node { - calculator: 'DropCalculator' - input_side_packet: "DROP_TIMESTAMPS:drop_timesamps" - input_stream: 'PACKET:out_1_sampled' - output_stream: 'PACKET:out_1' - } - )pb"); - } - - // Parse an absl::Time from RFC3339 format. - absl::Time ParseTime(const std::string& date_time_str) { - absl::Time result; - absl::ParseTime(absl::RFC3339_sec, date_time_str, &result, nullptr); - return result; - } - - // The point in simulated time when the test starts. - absl::Time StartTime() { return ParseTime("2020-11-03T20:00:00Z"); } - - // Initialize the test clock to follow simulated time. - void SetUpSimulationClock() { - auto executor = std::make_shared(8); - simulation_clock_ = executor->GetClock(); - clock_ = simulation_clock_.get(); - simulation_clock_->ThreadStart(); - clock_->SleepUntil(StartTime()); - simulation_clock_->ThreadFinish(); - MP_ASSERT_OK(graph_.SetExecutor("", executor)); - } - - // Initialize the test clock to follow wall time. - void SetUpRealClock() { clock_ = mediapipe::Clock::RealClock(); } - - // Create a few mediapipe input Packets holding ints. - void SetUpInputData() { - for (int i = 0; i < 100; ++i) { - input_packets_.push_back(MakePacket(i).At(Timestamp(i * 10000))); - } - } - - protected: - CalculatorGraph graph_; - mediapipe::Clock* clock_; - std::shared_ptr simulation_clock_; - std::vector input_packets_; - std::vector out_1_packets_; - std::vector allow_packets_; -}; - -// Shows that "FINISHED" can be indicated with either a packet or a timestamp -// bound. DropCalculator periodically drops one packet but always propagates -// the timestamp bound. Input packets are released or dropped promptly after -// each "FINISH" packet or a timestamp bound arrives. -TEST_F(FlowLimiterCalculatorTest, FinishedTimestamps) { - // Configure the test. - SetUpInputData(); - SetUpSimulationClock(); - CalculatorGraphConfig graph_config = InflightGraphConfig(); - auto limiter_options = ParseTextProtoOrDie(R"pb( - max_in_flight: 1 - max_in_queue: 1 - )pb"); - std::map side_packets = { - {"limiter_options", - MakePacket(limiter_options)}, - {"warmup_time", MakePacket(22000)}, - {"sleep_time", MakePacket(22000)}, - {"drop_timesamps", MakePacket(false)}, - {"clock", MakePacket(clock_)}, - }; - - // Start the graph. - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) { - out_1_packets_.push_back(p); - return absl::OkStatus(); - })); - MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) { - allow_packets_.push_back(p); - return absl::OkStatus(); - })); - simulation_clock_->ThreadStart(); - MP_ASSERT_OK(graph_.StartRun(side_packets)); - - // Add 9 input packets. - // 1. packet-0 is released, - // 2. packet-1 is queued, - // 3. packet-2 is queued and packet-1 is dropped, - // 4. packet-2 is released, and so forth. - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[0])); - clock_->Sleep(absl::Microseconds(1)); - EXPECT_EQ(allow_packets_.size(), 1); - EXPECT_EQ(allow_packets_.back().Get(), true); - clock_->Sleep(absl::Microseconds(10000)); - for (int i = 1; i < 8; i += 2) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - clock_->Sleep(absl::Microseconds(10000)); - EXPECT_EQ(allow_packets_.size(), i); - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i + 1])); - clock_->Sleep(absl::Microseconds(1)); - EXPECT_EQ(allow_packets_.size(), i + 1); - EXPECT_EQ(allow_packets_.back().Get(), false); - clock_->Sleep(absl::Microseconds(10000)); - EXPECT_EQ(allow_packets_.size(), i + 2); - EXPECT_EQ(allow_packets_.back().Get(), true); - } - - // Finish the graph. - MP_EXPECT_OK(graph_.CloseAllPacketSources()); - clock_->Sleep(absl::Microseconds(40000)); - MP_EXPECT_OK(graph_.WaitUntilDone()); - simulation_clock_->ThreadFinish(); - - // Validate the output. - // input_packets_[4] is dropped by the DropCalculator. - std::vector expected_output = {input_packets_[0], input_packets_[2], - input_packets_[6], input_packets_[8]}; - EXPECT_EQ(out_1_packets_, expected_output); -} - -// Shows that an output packet can be lost completely, and the -// FlowLimiterCalculator will stop waiting for it after in_flight_timeout. -// DropCalculator completely loses one packet including its timestamp bound. -// FlowLimiterCalculator waits 100 ms, and then starts releasing packets again. -TEST_F(FlowLimiterCalculatorTest, FinishedLost) { - // Configure the test. - SetUpInputData(); - SetUpSimulationClock(); - CalculatorGraphConfig graph_config = InflightGraphConfig(); - auto limiter_options = ParseTextProtoOrDie(R"pb( - max_in_flight: 1 - max_in_queue: 1 - in_flight_timeout: 100000 # 100 ms - )pb"); - std::map side_packets = { - {"limiter_options", - MakePacket(limiter_options)}, - {"warmup_time", MakePacket(22000)}, - {"sleep_time", MakePacket(22000)}, - {"drop_timesamps", MakePacket(true)}, - {"clock", MakePacket(clock_)}, - }; - - // Start the graph. - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) { - out_1_packets_.push_back(p); - return absl::OkStatus(); - })); - MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) { - allow_packets_.push_back(p); - return absl::OkStatus(); - })); - simulation_clock_->ThreadStart(); - MP_ASSERT_OK(graph_.StartRun(side_packets)); - - // Add 21 input packets. - // 1. packet-0 is released, packet-1 queued and dropped, and so forth. - // 2. packet-4 is lost by DropCalculator. - // 3. packet-5 through 13 are dropped while waiting for packet-4. - // 4. packet-4 expires and queued packet-14 is released. - // 5. packet-17, 19, and 20 are released on time. - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[0])); - clock_->Sleep(absl::Microseconds(10000)); - for (int i = 1; i < 21; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Finish the graph. - MP_EXPECT_OK(graph_.CloseAllPacketSources()); - clock_->Sleep(absl::Microseconds(40000)); - MP_EXPECT_OK(graph_.WaitUntilDone()); - simulation_clock_->ThreadFinish(); - - // Validate the output. - // input_packets_[4] is lost by the DropCalculator. - std::vector expected_output = { - input_packets_[0], input_packets_[2], input_packets_[14], - input_packets_[17], input_packets_[19], input_packets_[20], - }; - EXPECT_EQ(out_1_packets_, expected_output); -} - -// Shows what happens when a finish packet is delayed beyond in_flight_timeout. -// After in_flight_timeout, FlowLimiterCalculator continues releasing packets. -// Temporarily, more than max_in_flight frames are in flight. -// Eventually, the number of frames in flight returns to max_in_flight. -TEST_F(FlowLimiterCalculatorTest, FinishedDelayed) { - // Configure the test. - SetUpInputData(); - SetUpSimulationClock(); - CalculatorGraphConfig graph_config = InflightGraphConfig(); - auto limiter_options = ParseTextProtoOrDie(R"pb( - max_in_flight: 1 - max_in_queue: 1 - in_flight_timeout: 100000 # 100 ms - )pb"); - std::map side_packets = { - {"limiter_options", - MakePacket(limiter_options)}, - {"warmup_time", MakePacket(500000)}, - {"sleep_time", MakePacket(22000)}, - {"drop_timesamps", MakePacket(false)}, - {"clock", MakePacket(clock_)}, - }; - - // Start the graph. - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) { - out_1_packets_.push_back(p); - return absl::OkStatus(); - })); - MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) { - allow_packets_.push_back(p); - return absl::OkStatus(); - })); - simulation_clock_->ThreadStart(); - MP_ASSERT_OK(graph_.StartRun(side_packets)); - - // Add 71 input packets. - // 1. During the 500 ms WARMUP_TIME, the in_flight_timeout releases - // packets 0, 10, 20, 30, 40, 50, which are queued at the SleepCalculator. - // 2. During the next 120 ms, these 6 packets are processed. - // 3. After the graph is finally finished with warmup and the backlog packets, - // packets 60 through 70 are released and processed on time. - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[0])); - clock_->Sleep(absl::Microseconds(10000)); - for (int i = 1; i < 71; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Finish the graph. - MP_EXPECT_OK(graph_.CloseAllPacketSources()); - clock_->Sleep(absl::Microseconds(40000)); - MP_EXPECT_OK(graph_.WaitUntilDone()); - simulation_clock_->ThreadFinish(); - - // Validate the output. - // The graph is warming up or backlogged until packet 60. - std::vector expected_output = { - input_packets_[0], input_packets_[10], input_packets_[30], - input_packets_[40], input_packets_[50], input_packets_[60], - input_packets_[63], input_packets_[65], input_packets_[67], - input_packets_[69], input_packets_[70], - }; - EXPECT_EQ(out_1_packets_, expected_output); -} - -// Shows that packets on auxiliary input streams are relesed for the same -// timestamps as the main input stream, whether the auxiliary packets arrive -// early or late. -TEST_F(FlowLimiterCalculatorTest, TwoInputStreams) { - // Configure the test. - SetUpInputData(); - SetUpSimulationClock(); - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'in_1' - input_stream: 'in_2' - node { - calculator: 'FlowLimiterCalculator' - input_side_packet: 'OPTIONS:limiter_options' - input_stream: 'in_1' - input_stream: 'in_2' - input_stream: 'FINISHED:out_1' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_1_sampled' - output_stream: 'in_2_sampled' - output_stream: 'ALLOW:allow' - } - node { - calculator: 'SleepCalculator' - input_side_packet: 'WARMUP_TIME:warmup_time' - input_side_packet: 'SLEEP_TIME:sleep_time' - input_side_packet: 'CLOCK:clock' - input_stream: 'PACKET:in_1_sampled' - output_stream: 'PACKET:out_1_sampled' - } - node { - calculator: 'DropCalculator' - input_side_packet: "DROP_TIMESTAMPS:drop_timesamps" - input_stream: 'PACKET:out_1_sampled' - output_stream: 'PACKET:out_1' - } - )pb"); - - auto limiter_options = ParseTextProtoOrDie(R"pb( - max_in_flight: 1 - max_in_queue: 1 - in_flight_timeout: 100000 # 100 ms - )pb"); - std::map side_packets = { - {"limiter_options", - MakePacket(limiter_options)}, - {"warmup_time", MakePacket(22000)}, - {"sleep_time", MakePacket(22000)}, - {"drop_timesamps", MakePacket(true)}, - {"clock", MakePacket(clock_)}, - }; - - // Start the graph. - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) { - out_1_packets_.push_back(p); - return absl::OkStatus(); - })); - std::vector out_2_packets; - MP_EXPECT_OK(graph_.ObserveOutputStream("in_2_sampled", [&](Packet p) { - out_2_packets.push_back(p); - return absl::OkStatus(); - })); - MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) { - allow_packets_.push_back(p); - return absl::OkStatus(); - })); - simulation_clock_->ThreadStart(); - MP_ASSERT_OK(graph_.StartRun(side_packets)); - - // Add packets 0..9 to stream in_1, and packets 0..10 to stream in_2. - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[0])); - clock_->Sleep(absl::Microseconds(10000)); - for (int i = 1; i < 10; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_2", input_packets_[i - 1])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Add packets 10..20 to stream in_1, and packets 11..21 to stream in_2. - for (int i = 10; i < 21; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_2", input_packets_[i + 1])); - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Finish the graph run. - MP_EXPECT_OK(graph_.CloseAllPacketSources()); - clock_->Sleep(absl::Microseconds(40000)); - MP_EXPECT_OK(graph_.WaitUntilDone()); - simulation_clock_->ThreadFinish(); - - // Validate the output. - // Packet input_packets_[4] is lost by the DropCalculator. - std::vector expected_output = { - input_packets_[0], input_packets_[2], input_packets_[14], - input_packets_[17], input_packets_[19], input_packets_[20], - }; - EXPECT_EQ(out_1_packets_, expected_output); - // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled. - std::vector expected_output_2 = { - input_packets_[0], input_packets_[2], input_packets_[4], - input_packets_[14], input_packets_[17], input_packets_[19], - input_packets_[20], - }; - EXPECT_EQ(out_2_packets, expected_output_2); -} - -// Shows how FlowLimiterCalculator releases packets with max_in_queue 0. -// Shows how auxiliary input streams still work with max_in_queue 0. -// The processing time "sleep_time" is reduced from 22ms to 12ms to create -// the same frame rate as FlowLimiterCalculatorTest::TwoInputStreams. -TEST_F(FlowLimiterCalculatorTest, ZeroQueue) { - // Configure the test. - SetUpInputData(); - SetUpSimulationClock(); - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'in_1' - input_stream: 'in_2' - node { - calculator: 'FlowLimiterCalculator' - input_side_packet: 'OPTIONS:limiter_options' - input_stream: 'in_1' - input_stream: 'in_2' - input_stream: 'FINISHED:out_1' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_1_sampled' - output_stream: 'in_2_sampled' - output_stream: 'ALLOW:allow' - } - node { - calculator: 'SleepCalculator' - input_side_packet: 'WARMUP_TIME:warmup_time' - input_side_packet: 'SLEEP_TIME:sleep_time' - input_side_packet: 'CLOCK:clock' - input_stream: 'PACKET:in_1_sampled' - output_stream: 'PACKET:out_1_sampled' - } - node { - calculator: 'DropCalculator' - input_side_packet: "DROP_TIMESTAMPS:drop_timesamps" - input_stream: 'PACKET:out_1_sampled' - output_stream: 'PACKET:out_1' - } - )pb"); - - auto limiter_options = ParseTextProtoOrDie(R"pb( - max_in_flight: 1 - max_in_queue: 0 - in_flight_timeout: 100000 # 100 ms - )pb"); - std::map side_packets = { - {"limiter_options", - MakePacket(limiter_options)}, - {"warmup_time", MakePacket(12000)}, - {"sleep_time", MakePacket(12000)}, - {"drop_timesamps", MakePacket(true)}, - {"clock", MakePacket(clock_)}, - }; - - // Start the graph. - MP_ASSERT_OK(graph_.Initialize(graph_config)); - MP_EXPECT_OK(graph_.ObserveOutputStream("out_1", [this](Packet p) { - out_1_packets_.push_back(p); - return absl::OkStatus(); - })); - std::vector out_2_packets; - MP_EXPECT_OK(graph_.ObserveOutputStream("in_2_sampled", [&](Packet p) { - out_2_packets.push_back(p); - return absl::OkStatus(); - })); - MP_EXPECT_OK(graph_.ObserveOutputStream("allow", [this](Packet p) { - allow_packets_.push_back(p); - return absl::OkStatus(); - })); - simulation_clock_->ThreadStart(); - MP_ASSERT_OK(graph_.StartRun(side_packets)); - - // Add packets 0..9 to stream in_1, and packets 0..10 to stream in_2. - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[0])); - clock_->Sleep(absl::Microseconds(10000)); - for (int i = 1; i < 10; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_2", input_packets_[i - 1])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Add packets 10..20 to stream in_1, and packets 11..21 to stream in_2. - for (int i = 10; i < 21; ++i) { - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_2", input_packets_[i + 1])); - MP_EXPECT_OK(graph_.AddPacketToInputStream("in_1", input_packets_[i])); - clock_->Sleep(absl::Microseconds(10000)); - } - - // Finish the graph run. - MP_EXPECT_OK(graph_.CloseAllPacketSources()); - clock_->Sleep(absl::Microseconds(40000)); - MP_EXPECT_OK(graph_.WaitUntilDone()); - simulation_clock_->ThreadFinish(); - - // Validate the output. - // Packet input_packets_[4] is lost by the DropCalculator. - std::vector expected_output = { - input_packets_[0], input_packets_[2], input_packets_[15], - input_packets_[17], input_packets_[19], - }; - EXPECT_EQ(out_1_packets_, expected_output); - // Exactly the timestamps released by FlowLimiterCalculator for in_1_sampled. - std::vector expected_output_2 = { - input_packets_[0], input_packets_[2], input_packets_[4], - input_packets_[15], input_packets_[17], input_packets_[19], - }; - EXPECT_EQ(out_2_packets, expected_output_2); -} - -} // anonymous namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/gate_calculator.cc b/mediapipe/calculators/core/gate_calculator.cc deleted file mode 100644 index 189671860..000000000 --- a/mediapipe/calculators/core/gate_calculator.cc +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2019-2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/gate_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/header_util.h" - -namespace mediapipe { - -namespace { -enum GateState { - GATE_UNINITIALIZED, - GATE_ALLOW, - GATE_DISALLOW, -}; - -std::string ToString(GateState state) { - switch (state) { - case GATE_UNINITIALIZED: - return "UNINITIALIZED"; - case GATE_ALLOW: - return "ALLOW"; - case GATE_DISALLOW: - return "DISALLOW"; - } - DLOG(FATAL) << "Unknown GateState"; - return "UNKNOWN"; -} -} // namespace - -// Controls whether or not the input packets are passed further along the graph. -// Takes multiple data input streams and either an ALLOW or a DISALLOW control -// input stream. It outputs an output stream for each input stream that is not -// ALLOW or DISALLOW as well as an optional STATE_CHANGE stream which downstream -// calculators can use to respond to state-change events. -// -// If the current ALLOW packet is set to true, the input packets are passed to -// their corresponding output stream unchanged. If the ALLOW packet is set to -// false, the current input packet is NOT passed to the output stream. If using -// DISALLOW, the behavior is opposite of ALLOW. -// -// By default, an empty packet in the ALLOW or DISALLOW input stream indicates -// disallowing the corresponding packets in other input streams. The behavior -// can be inverted with a calculator option. -// -// ALLOW or DISALLOW can also be specified as an input side packet. The rules -// for evaluation remain the same as above. -// -// ALLOW/DISALLOW inputs must be specified either using input stream or -// via input side packet but not both. -// -// Intended to be used with the default input stream handler, which synchronizes -// all data input streams with the ALLOW/DISALLOW control input stream. -// -// Example config: -// node { -// calculator: "GateCalculator" -// input_side_packet: "ALLOW:allow" or "DISALLOW:disallow" -// input_stream: "input_stream0" -// input_stream: "input_stream1" -// input_stream: "input_streamN" -// input_stream: "ALLOW:allow" or "DISALLOW:disallow" -// output_stream: "STATE_CHANGE:state_change" -// output_stream: "output_stream0" -// output_stream: "output_stream1" -// output_stream: "output_streamN" -// } -class GateCalculator : public CalculatorBase { - public: - GateCalculator() {} - - static absl::Status CheckAndInitAllowDisallowInputs(CalculatorContract* cc) { - bool input_via_side_packet = cc->InputSidePackets().HasTag("ALLOW") || - cc->InputSidePackets().HasTag("DISALLOW"); - bool input_via_stream = - cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW"); - // Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW - // input. - RET_CHECK(input_via_side_packet ^ input_via_stream); - - if (input_via_side_packet) { - RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^ - cc->InputSidePackets().HasTag("DISALLOW")); - - if (cc->InputSidePackets().HasTag("ALLOW")) { - cc->InputSidePackets().Tag("ALLOW").Set(); - } else { - cc->InputSidePackets().Tag("DISALLOW").Set(); - } - } else { - RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW")); - - if (cc->Inputs().HasTag("ALLOW")) { - cc->Inputs().Tag("ALLOW").Set(); - } else { - cc->Inputs().Tag("DISALLOW").Set(); - } - } - return absl::OkStatus(); - } - - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK_OK(CheckAndInitAllowDisallowInputs(cc)); - - const int num_data_streams = cc->Inputs().NumEntries(""); - RET_CHECK_GE(num_data_streams, 1); - RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams) - << "Number of data output streams must match with data input streams."; - - for (int i = 0; i < num_data_streams; ++i) { - cc->Inputs().Get("", i).SetAny(); - cc->Outputs().Get("", i).SetSameAs(&cc->Inputs().Get("", i)); - } - - if (cc->Outputs().HasTag("STATE_CHANGE")) { - cc->Outputs().Tag("STATE_CHANGE").Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - use_side_packet_for_allow_disallow_ = false; - if (cc->InputSidePackets().HasTag("ALLOW")) { - use_side_packet_for_allow_disallow_ = true; - allow_by_side_packet_decision_ = - cc->InputSidePackets().Tag("ALLOW").Get(); - } else if (cc->InputSidePackets().HasTag("DISALLOW")) { - use_side_packet_for_allow_disallow_ = true; - allow_by_side_packet_decision_ = - !cc->InputSidePackets().Tag("DISALLOW").Get(); - } - - cc->SetOffset(TimestampDiff(0)); - num_data_streams_ = cc->Inputs().NumEntries(""); - last_gate_state_ = GATE_UNINITIALIZED; - RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs())); - - const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>(); - empty_packets_as_allow_ = options.empty_packets_as_allow(); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - bool allow = empty_packets_as_allow_; - if (use_side_packet_for_allow_disallow_) { - allow = allow_by_side_packet_decision_; - } else { - if (cc->Inputs().HasTag("ALLOW") && - !cc->Inputs().Tag("ALLOW").IsEmpty()) { - allow = cc->Inputs().Tag("ALLOW").Get(); - } - if (cc->Inputs().HasTag("DISALLOW") && - !cc->Inputs().Tag("DISALLOW").IsEmpty()) { - allow = !cc->Inputs().Tag("DISALLOW").Get(); - } - } - const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW; - - if (cc->Outputs().HasTag("STATE_CHANGE")) { - if (last_gate_state_ != GATE_UNINITIALIZED && - last_gate_state_ != new_gate_state) { - VLOG(2) << "State transition in " << cc->NodeName() << " @ " - << cc->InputTimestamp().Value() << " from " - << ToString(last_gate_state_) << " to " - << ToString(new_gate_state); - cc->Outputs() - .Tag("STATE_CHANGE") - .AddPacket(MakePacket(allow).At(cc->InputTimestamp())); - } - } - last_gate_state_ = new_gate_state; - - if (!allow) { - // Close the output streams if the gate will be permanently closed. - // Prevents buffering in calculators whose parents do no use SetOffset. - for (int i = 0; i < num_data_streams_; ++i) { - if (!cc->Outputs().Get("", i).IsClosed() && - use_side_packet_for_allow_disallow_) { - cc->Outputs().Get("", i).Close(); - } - } - return absl::OkStatus(); - } - - // Process data streams. - for (int i = 0; i < num_data_streams_; ++i) { - if (!cc->Inputs().Get("", i).IsEmpty()) { - cc->Outputs().Get("", i).AddPacket(cc->Inputs().Get("", i).Value()); - } - } - - return absl::OkStatus(); - } - - private: - GateState last_gate_state_ = GATE_UNINITIALIZED; - int num_data_streams_; - bool empty_packets_as_allow_; - bool use_side_packet_for_allow_disallow_; - bool allow_by_side_packet_decision_; -}; -REGISTER_CALCULATOR(GateCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/gate_calculator.proto b/mediapipe/calculators/core/gate_calculator.proto deleted file mode 100644 index 76bacc74e..000000000 --- a/mediapipe/calculators/core/gate_calculator.proto +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2019-2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message GateCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional GateCalculatorOptions ext = 261754847; - } - - // By default an empty packet in the ALLOW or DISALLOW input stream indicates - // disallowing the corresponding packets in the data input streams. Setting - // this option to true inverts that, allowing the data packets to go through. - optional bool empty_packets_as_allow = 1; -} diff --git a/mediapipe/calculators/core/gate_calculator_test.cc b/mediapipe/calculators/core/gate_calculator_test.cc deleted file mode 100644 index 0b78b9b75..000000000 --- a/mediapipe/calculators/core/gate_calculator_test.cc +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2019-2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -class GateCalculatorTest : public ::testing::Test { - protected: - // Helper to run a graph and return status. - static absl::Status RunGraph(const std::string& proto) { - auto runner = absl::make_unique( - ParseTextProtoOrDie(proto)); - return runner->Run(); - } - - // Use this when ALLOW/DISALLOW input is provided as a side packet. - void RunTimeStep(int64 timestamp, bool stream_payload) { - runner_->MutableInputs()->Get("", 0).packets.push_back( - MakePacket(stream_payload).At(Timestamp(timestamp))); - MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed."; - } - - // Use this when ALLOW/DISALLOW input is provided as an input stream. - void RunTimeStep(int64 timestamp, const std::string& control_tag, - bool control) { - runner_->MutableInputs()->Get("", 0).packets.push_back( - MakePacket(true).At(Timestamp(timestamp))); - runner_->MutableInputs() - ->Tag(control_tag) - .packets.push_back(MakePacket(control).At(Timestamp(timestamp))); - MP_ASSERT_OK(runner_->Run()) << "Calculator execution failed."; - } - - void SetRunner(const std::string& proto) { - runner_ = absl::make_unique( - ParseTextProtoOrDie(proto)); - } - - CalculatorRunner* runner() { return runner_.get(); } - - private: - std::unique_ptr runner_; -}; - -TEST_F(GateCalculatorTest, InvalidInputs) { - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - input_stream: "DISALLOW:gating_stream" - output_stream: "test_output" - )"))); - - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_side_packet: "ALLOW:gating_stream" - input_side_packet: "DISALLOW:gating_stream" - output_stream: "test_output" - )"))); - - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - input_side_packet: "ALLOW:gating_stream" - output_stream: "test_output" - )"))); - - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "DISALLOW:gating_stream" - input_side_packet: "DISALLOW:gating_stream" - output_stream: "test_output" - )"))); - - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - input_side_packet: "DISALLOW:gating_stream" - output_stream: "test_output" - )"))); - - EXPECT_TRUE(absl::IsInternal(GateCalculatorTest::RunGraph(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "DISALLOW:gating_stream" - input_side_packet: "ALLOW:gating_stream" - output_stream: "test_output" - )"))); -} - -TEST_F(GateCalculatorTest, AllowByALLOWSidePacketSetToTrue) { - SetRunner(R"( - calculator: "GateCalculator" - input_side_packet: "ALLOW:gating_stream" - input_stream: "test_input" - output_stream: "test_output" - )"); - runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(true)); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue1, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); - EXPECT_EQ(false, output[1].Get()); -} - -TEST_F(GateCalculatorTest, AllowByDisallowSidePacketSetToFalse) { - SetRunner(R"( - calculator: "GateCalculator" - input_side_packet: "DISALLOW:gating_stream" - input_stream: "test_input" - output_stream: "test_output" - )"); - runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(false)); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue1, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); - EXPECT_EQ(false, output[1].Get()); -} - -TEST_F(GateCalculatorTest, DisallowByALLOWSidePacketSetToFalse) { - SetRunner(R"( - calculator: "GateCalculator" - input_side_packet: "ALLOW:gating_stream" - input_stream: "test_input" - output_stream: "test_output" - )"); - runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false)); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(0, output.size()); -} - -TEST_F(GateCalculatorTest, DisallowByDISALLOWSidePacketSetToTrue) { - SetRunner(R"( - calculator: "GateCalculator" - input_side_packet: "DISALLOW:gating_stream" - input_stream: "test_input" - output_stream: "test_output" - )"); - runner()->MutableSidePackets()->Tag("DISALLOW") = Adopt(new bool(true)); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(0, output.size()); -} - -TEST_F(GateCalculatorTest, Allow) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - output_stream: "test_output" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "ALLOW", true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, "ALLOW", false); - constexpr int64 kTimestampValue2 = 44; - RunTimeStep(kTimestampValue2, "ALLOW", true); - constexpr int64 kTimestampValue3 = 45; - RunTimeStep(kTimestampValue3, "ALLOW", false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue0, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue2, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); - EXPECT_EQ(true, output[1].Get()); -} - -TEST_F(GateCalculatorTest, Disallow) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "DISALLOW:gating_stream" - output_stream: "test_output" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "DISALLOW", true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, "DISALLOW", false); - constexpr int64 kTimestampValue2 = 44; - RunTimeStep(kTimestampValue2, "DISALLOW", true); - constexpr int64 kTimestampValue3 = 45; - RunTimeStep(kTimestampValue3, "DISALLOW", false); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); - EXPECT_EQ(true, output[1].Get()); -} - -TEST_F(GateCalculatorTest, AllowWithStateChange) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - output_stream: "test_output" - output_stream: "STATE_CHANGE:state_changed" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "ALLOW", false); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, "ALLOW", true); - constexpr int64 kTimestampValue2 = 44; - RunTimeStep(kTimestampValue2, "ALLOW", true); - constexpr int64 kTimestampValue3 = 45; - RunTimeStep(kTimestampValue3, "ALLOW", false); - - const std::vector& output = - runner()->Outputs().Get("STATE_CHANGE", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); // Allow. - EXPECT_EQ(false, output[1].Get()); // Disallow. -} - -TEST_F(GateCalculatorTest, DisallowWithStateChange) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "DISALLOW:gating_stream" - output_stream: "test_output" - output_stream: "STATE_CHANGE:state_changed" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "DISALLOW", true); - constexpr int64 kTimestampValue1 = 43; - RunTimeStep(kTimestampValue1, "DISALLOW", false); - constexpr int64 kTimestampValue2 = 44; - RunTimeStep(kTimestampValue2, "DISALLOW", false); - constexpr int64 kTimestampValue3 = 45; - RunTimeStep(kTimestampValue3, "DISALLOW", true); - - const std::vector& output = - runner()->Outputs().Get("STATE_CHANGE", 0).packets; - ASSERT_EQ(2, output.size()); - EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value()); - EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value()); - EXPECT_EQ(true, output[0].Get()); // Allow. - EXPECT_EQ(false, output[1].Get()); // Disallow. -} - -// Must not detect disallow value for first timestamp as a state change. -TEST_F(GateCalculatorTest, DisallowInitialNoStateTransition) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "DISALLOW:gating_stream" - output_stream: "test_output" - output_stream: "STATE_CHANGE:state_changed" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "DISALLOW", false); - - const std::vector& output = - runner()->Outputs().Get("STATE_CHANGE", 0).packets; - ASSERT_EQ(0, output.size()); -} - -// Must not detect allow value for first timestamp as a state change. -TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - output_stream: "test_output" - output_stream: "STATE_CHANGE:state_changed" - )"); - - constexpr int64 kTimestampValue0 = 42; - RunTimeStep(kTimestampValue0, "ALLOW", true); - - const std::vector& output = - runner()->Outputs().Get("STATE_CHANGE", 0).packets; - ASSERT_EQ(0, output.size()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/immediate_mux_calculator.cc b/mediapipe/calculators/core/immediate_mux_calculator.cc deleted file mode 100644 index 0e51cda5e..000000000 --- a/mediapipe/calculators/core/immediate_mux_calculator.cc +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// This Calculator multiplexes several input streams into a single -// output stream, dropping input packets with timestamps older than the -// last output packet. In case two packets arrive with the same timestamp, -// the packet with the lower stream index will be output and the rest will -// be dropped. -// -// This Calculator optionally produces a finish inidicator as its second -// output stream. One indicator packet is produced for each input packet -// received. -// -// This Calculator can be used with an ImmediateInputStreamHandler or with the -// default ISH. -// -// This Calculator is designed to work with a Demux calculator such as -// the RoundRobinDemuxCalculator. Therefore, packets from different -// input streams are normally not expected to have the same timestamp. -// -// NOTE: this calculator can drop packets non-deterministically, depending on -// how fast the input streams are fed. In most cases, MuxCalculator should be -// preferred. In particular, dropping packets can interfere with rate limiting -// mechanisms. -class ImmediateMuxCalculator : public CalculatorBase { - public: - // This calculator combines any set of input streams into a single - // output stream. All input stream types must match the output stream type. - static absl::Status GetContract(CalculatorContract* cc); - - // Passes any input packet to the output stream immediately, unless the - // packet timestamp is lower than a previously passed packet. - absl::Status Process(CalculatorContext* cc) override; - absl::Status Open(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(ImmediateMuxCalculator); - -absl::Status ImmediateMuxCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Outputs().NumEntries() >= 1 && cc->Outputs().NumEntries() <= 2) - << "This calculator produces only one or two output streams."; - cc->Outputs().Index(0).SetAny(); - if (cc->Outputs().NumEntries() >= 2) { - cc->Outputs().Index(1).Set(); - } - for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - cc->Inputs().Index(i).SetSameAs(&cc->Outputs().Index(0)); - } - return absl::OkStatus(); -} - -absl::Status ImmediateMuxCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) { - // Pass along the first packet, unless it has been superseded. - for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - const Packet& packet = cc->Inputs().Index(i).Value(); - if (!packet.IsEmpty()) { - if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) { - cc->Outputs().Index(0).AddPacket(packet); - } else { - LOG_FIRST_N(WARNING, 5) - << "Dropping a packet with timestamp " << packet.Timestamp(); - } - if (cc->Outputs().NumEntries() >= 2) { - Timestamp output_timestamp = std::max( - cc->InputTimestamp(), cc->Outputs().Index(1).NextTimestampBound()); - cc->Outputs().Index(1).Add(new bool(true), output_timestamp); - } - } - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/immediate_mux_calculator_test.cc b/mediapipe/calculators/core/immediate_mux_calculator_test.cc deleted file mode 100644 index 6913fd000..000000000 --- a/mediapipe/calculators/core/immediate_mux_calculator_test.cc +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "absl/synchronization/mutex.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/port/threadpool.h" -#include "mediapipe/framework/tool/sink.h" - -// Tests for ImmediateMuxCalculator. These tests show how parallel output -// packets are handled when they arrive in various orders. -using testing::ElementsAre; - -namespace mediapipe { - -namespace { - -// A simple Semaphore for synchronizing test threads. -class AtomicSemaphore { - public: - AtomicSemaphore(int64_t supply) : supply_(supply) {} - void Acquire(int64_t amount) { - while (supply_.fetch_sub(amount) - amount < 0) { - Release(amount); - } - } - void Release(int64_t amount) { supply_.fetch_add(amount); } - - private: - std::atomic supply_; -}; - -// A mediapipe::Executor that signals the start and finish of each task. -// Provides 4 worker threads. -class CountingExecutor : public Executor { - public: - CountingExecutor(std::function start_callback, - std::function finish_callback) - : thread_pool_(4), - start_callback_(std::move(start_callback)), - finish_callback_(std::move(finish_callback)) { - thread_pool_.StartWorkers(); - } - void Schedule(std::function task) override { - start_callback_(); - thread_pool_.Schedule([this, task] { - task(); - finish_callback_(); - }); - } - - private: - ThreadPool thread_pool_; - std::function start_callback_; - std::function finish_callback_; -}; - -// Returns a new mediapipe::Executor with 4 worker threads. -std::shared_ptr MakeExecutor(std::function start_callback, - std::function finish_callback) { - return std::make_shared(start_callback, finish_callback); -} - -// Tests showing ImmediateMuxCalculator dropping packets in various sequences. -class ImmediateMuxCalculatorTest : public ::testing::Test { - protected: - void SetUpMuxGraph() { - ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"( - input_stream: "input_packets_0" - input_stream: "input_packets_1" - node { - calculator: "ImmediateMuxCalculator" - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - input_stream: "input_packets_0" - input_stream: "input_packets_1" - output_stream: "output_packets_0" - } - )", - &graph_config_)); - } - - void SetUpDemuxGraph() { - ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"( - input_stream: "input_packets_0" - node { - calculator: "RoundRobinDemuxCalculator" - input_stream: "input_packets_0" - output_stream: "OUTPUT:0:input_0" - output_stream: "OUTPUT:1:input_1" - } - node { - calculator: "LambdaCalculator" - input_side_packet: 'callback_0' - input_stream: "input_0" - output_stream: "output_0" - } - node { - calculator: "LambdaCalculator" - input_side_packet: 'callback_1' - input_stream: "input_1" - output_stream: "output_1" - } - node { - calculator: "ImmediateMuxCalculator" - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - input_stream: "output_0" - input_stream: "output_1" - output_stream: "output_packets_0" - } - )", - &graph_config_)); - } - - void SetUpDemuxInFlightGraph() { - ASSERT_TRUE(proto_ns::TextFormat::ParseFromString(R"( - input_stream: "input_packets_0" - node { - calculator: 'FlowLimiterCalculator' - input_stream_handler { - input_stream_handler: 'ImmediateInputStreamHandler' - } - input_side_packet: 'MAX_IN_FLIGHT:max_in_flight' - input_stream: 'input_packets_0' - input_stream: 'FINISHED:finish_indicator' - input_stream_info: { - tag_index: 'FINISHED' - back_edge: true - } - output_stream: 'input_0_sampled' - } - node { - calculator: "RoundRobinDemuxCalculator" - input_stream: "input_0_sampled" - output_stream: "OUTPUT:0:input_0" - output_stream: "OUTPUT:1:input_1" - } - node { - calculator: "LambdaCalculator" - input_side_packet: 'callback_0' - input_stream: "input_0" - output_stream: "output_0" - } - node { - calculator: "LambdaCalculator" - input_side_packet: 'callback_1' - input_stream: "input_1" - output_stream: "output_1" - } - node { - calculator: "ImmediateMuxCalculator" - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - input_stream: "output_0" - input_stream: "output_1" - output_stream: 'output_packets_0' - output_stream: 'finish_indicator' - } - )", - &graph_config_)); - } - - static Packet PacketAt(int64 ts) { - return Adopt(new int64(999)).At(Timestamp(ts)); - } - static Packet None() { return Packet().At(Timestamp::OneOverPostStream()); } - static bool IsNone(const Packet& packet) { - return packet.Timestamp() == Timestamp::OneOverPostStream(); - } - // Return the values of the timestamps of a vector of Packets. - static std::vector TimestampValues( - const std::vector& packets) { - std::vector result; - for (const Packet& p : packets) { - result.push_back(p.Timestamp().Value()); - } - return result; - } - - // Runs a CalculatorGraph with a series of packet sets. - // Returns a vector of packets from each graph output stream. - void RunGraph(const std::vector>& input_sets, - std::vector* output_packets) { - // Register output packet observers. - tool::AddVectorSink("output_packets_0", &graph_config_, output_packets); - - // Start running the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config_)); - MP_ASSERT_OK(graph.StartRun({})); - - // Send each packet to the graph in the specified order. - for (int t = 0; t < input_sets.size(); t++) { - const std::vector& input_set = input_sets[t]; - MP_EXPECT_OK(graph.WaitUntilIdle()); - for (int i = 0; i < input_set.size(); i++) { - const Packet& packet = input_set[i]; - if (!IsNone(packet)) { - MP_EXPECT_OK(graph.AddPacketToInputStream( - absl::StrCat("input_packets_", i), packet)); - } - } - } - MP_ASSERT_OK(graph.CloseAllInputStreams()); - MP_ASSERT_OK(graph.WaitUntilDone()); - } - - CalculatorGraphConfig graph_config_; -}; - -TEST_F(ImmediateMuxCalculatorTest, IncreasingTimestamps) { - // Run the graph with a series of packet sets. - std::vector> input_sets = { - {PacketAt(10000), None()}, // - {PacketAt(20000), None()}, // - {None(), PacketAt(30000)}, // - {None(), PacketAt(40000)}, - }; - SetUpMuxGraph(); - std::vector output_packets; - RunGraph(input_sets, &output_packets); - - // Validate the output packets. - EXPECT_THAT(TimestampValues(output_packets), - ElementsAre(10000, 20000, 30000, 40000)); -} - -TEST_F(ImmediateMuxCalculatorTest, SupersededTimestamp) { - // Run the graph with a series of packet sets. - std::vector> input_sets = { - {PacketAt(10000), None()}, // - {PacketAt(30000), None()}, // - {None(), PacketAt(20000)}, // - {None(), PacketAt(40000)}, - }; - SetUpMuxGraph(); - std::vector output_packets; - RunGraph(input_sets, &output_packets); - - // Output packet 20000 is superseded and dropped. - EXPECT_THAT(TimestampValues(output_packets), - ElementsAre(10000, 30000, 40000)); -} - -TEST_F(ImmediateMuxCalculatorTest, SimultaneousTimestamps) { - // Run the graph with a series of packet sets. - std::vector> input_sets = { - {PacketAt(10000), None()}, // - {PacketAt(40000), PacketAt(20000)}, // - {None(), PacketAt(30000)}, - }; - SetUpMuxGraph(); - std::vector output_packets; - RunGraph(input_sets, &output_packets); - - // Output packet 20000 is superseded and dropped. - EXPECT_THAT(TimestampValues(output_packets), ElementsAre(10000, 40000)); -} - -// A Calculator::Process callback function. -typedef std::function - ProcessFunction; - -// A testing callback function that passes through all packets. -absl::Status PassThrough(const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - for (int i = 0; i < inputs.NumEntries(); ++i) { - if (!inputs.Index(i).Value().IsEmpty()) { - outputs->Index(i).AddPacket(inputs.Index(i).Value()); - } - } - return absl::OkStatus(); -} - -TEST_F(ImmediateMuxCalculatorTest, Demux) { - // Semaphores to sequence the parallel Process outputs. - AtomicSemaphore semaphore_0(0); - AtomicSemaphore semaphore_1(0); - ProcessFunction wait_0 = [&semaphore_0](const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - semaphore_0.Acquire(1); - return PassThrough(inputs, outputs); - }; - ProcessFunction wait_1 = [&semaphore_1](const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - semaphore_1.Acquire(1); - return PassThrough(inputs, outputs); - }; - - // A callback to await and capture output packets. - std::vector out_packets; - absl::Mutex out_mutex; - auto out_cb = [&](const Packet& p) { - absl::MutexLock lock(&out_mutex); - out_packets.push_back(p); - return absl::OkStatus(); - }; - auto wait_for = [&](std::function cond) { - absl::MutexLock lock(&out_mutex); - out_mutex.Await(absl::Condition(&cond)); - }; - SetUpDemuxGraph(); - - // Start the graph and add five input packets. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config_, - { - {"callback_0", Adopt(new auto(wait_0))}, - {"callback_1", Adopt(new auto(wait_1))}, - })); - MP_ASSERT_OK(graph.ObserveOutputStream("output_packets_0", out_cb)); - MP_ASSERT_OK(graph.StartRun({})); - MP_EXPECT_OK( - graph.AddPacketToInputStream("input_packets_0", PacketAt(10000))); - MP_EXPECT_OK( - graph.AddPacketToInputStream("input_packets_0", PacketAt(20000))); - MP_EXPECT_OK( - graph.AddPacketToInputStream("input_packets_0", PacketAt(30000))); - MP_EXPECT_OK( - graph.AddPacketToInputStream("input_packets_0", PacketAt(40000))); - MP_EXPECT_OK( - graph.AddPacketToInputStream("input_packets_0", PacketAt(50000))); - - // Release the outputs in order 20000, 10000, 30000, 50000, 40000. - semaphore_1.Release(1); // 20000 - wait_for([&] { return !out_packets.empty(); }); - semaphore_0.Release(1); // 10000 - semaphore_0.Release(1); // 30000 - wait_for([&] { return out_packets.size() >= 2; }); - semaphore_0.Release(1); // 50000 - wait_for([&] { return out_packets.size() >= 3; }); - semaphore_1.Release(1); // 40000 - MP_ASSERT_OK(graph.CloseAllInputStreams()); - MP_ASSERT_OK(graph.WaitUntilDone()); - - // Output packets 10000 and 40000 are superseded and dropped. - EXPECT_THAT(TimestampValues(out_packets), ElementsAre(20000, 30000, 50000)); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/make_pair_calculator.cc b/mediapipe/calculators/core/make_pair_calculator.cc deleted file mode 100644 index 561656861..000000000 --- a/mediapipe/calculators/core/make_pair_calculator.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace api2 { - -// Given two input streams (A, B), output a single stream containing a pair. -// -// Example config: -// node { -// calculator: "MakePairCalculator" -// input_stream: "packet_a" -// input_stream: "packet_b" -// output_stream: "output_pair_a_b" -// } -class MakePairCalculator : public Node { - public: - static constexpr Input::Multiple kIn{""}; - // Note that currently api2::Packet is a different type from mediapipe::Packet - static constexpr Output> - kPair{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kPair); - - static absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK_EQ(kIn(cc).Count(), 2); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - kPair(cc).Send({kIn(cc)[0].packet(), kIn(cc)[1].packet()}); - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(MakePairCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_multiply_calculator.cc b/mediapipe/calculators/core/matrix_multiply_calculator.cc deleted file mode 100644 index d52e0c2fa..000000000 --- a/mediapipe/calculators/core/matrix_multiply_calculator.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Eigen/Core" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace api2 { -// Perform a (left) matrix multiply. Meaning (output = A * input) -// where A is the matrix which is provided as an input side packet. -// -// Example config: -// node { -// calculator: "MatrixMultiplyCalculator" -// input_stream: "samples" -// output_stream: "multiplied_samples" -// input_side_packet: "multiplication_matrix" -// } -class MatrixMultiplyCalculator : public Node { - public: - static constexpr Input kIn{""}; - static constexpr Output kOut{""}; - static constexpr SideInput kSide{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kSide); - - absl::Status Process(CalculatorContext* cc) override; -}; -MEDIAPIPE_REGISTER_NODE(MatrixMultiplyCalculator); - -absl::Status MatrixMultiplyCalculator::Process(CalculatorContext* cc) { - kOut(cc).Send(*kSide(cc) * *kIn(cc)); - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc b/mediapipe/calculators/core/matrix_multiply_calculator_test.cc deleted file mode 100644 index e62ca8073..000000000 --- a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "Eigen/Core" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { -namespace { - -// A 3x4 Matrix of random integers in [0,1000). -const char kMatrixText[] = - "rows: 3\n" - "cols: 4\n" - "packed_data: 387\n" - "packed_data: 940\n" - "packed_data: 815\n" - "packed_data: 825\n" - "packed_data: 997\n" - "packed_data: 884\n" - "packed_data: 419\n" - "packed_data: 763\n" - "packed_data: 123\n" - "packed_data: 30\n" - "packed_data: 825\n" - "packed_data: 299\n"; - -// A 4x20 Matrix of random integers in [0,10). -// Each column of this matrix is a sample. -const char kSamplesText[] = - "rows: 4\n" - "cols: 20\n" - "packed_data: 7\n" - "packed_data: 9\n" - "packed_data: 5\n" - "packed_data: 9\n" - "packed_data: 6\n" - "packed_data: 3\n" - "packed_data: 0\n" - "packed_data: 7\n" - "packed_data: 1\n" - "packed_data: 3\n" - "packed_data: 3\n" - "packed_data: 2\n" - "packed_data: 4\n" - "packed_data: 5\n" - "packed_data: 0\n" - "packed_data: 4\n" - "packed_data: 6\n" - "packed_data: 0\n" - "packed_data: 1\n" - "packed_data: 2\n" - "packed_data: 0\n" - "packed_data: 2\n" - "packed_data: 0\n" - "packed_data: 3\n" - "packed_data: 1\n" - "packed_data: 7\n" - "packed_data: 4\n" - "packed_data: 9\n" - "packed_data: 8\n" - "packed_data: 8\n" - "packed_data: 6\n" - "packed_data: 4\n" - "packed_data: 6\n" - "packed_data: 8\n" - "packed_data: 1\n" - "packed_data: 9\n" - "packed_data: 7\n" - "packed_data: 5\n" - "packed_data: 3\n" - "packed_data: 5\n" - "packed_data: 3\n" - "packed_data: 5\n" - "packed_data: 7\n" - "packed_data: 7\n" - "packed_data: 3\n" - "packed_data: 3\n" - "packed_data: 6\n" - "packed_data: 4\n" - "packed_data: 7\n" - "packed_data: 7\n" - "packed_data: 2\n" - "packed_data: 5\n" - "packed_data: 4\n" - "packed_data: 8\n" - "packed_data: 1\n" - "packed_data: 0\n" - "packed_data: 2\n" - "packed_data: 0\n" - "packed_data: 3\n" - "packed_data: 4\n" - "packed_data: 6\n" - "packed_data: 6\n" - "packed_data: 8\n" - "packed_data: 5\n" - "packed_data: 5\n" - "packed_data: 8\n" - "packed_data: 9\n" - "packed_data: 7\n" - "packed_data: 3\n" - "packed_data: 7\n" - "packed_data: 2\n" - "packed_data: 7\n" - "packed_data: 8\n" - "packed_data: 2\n" - "packed_data: 1\n" - "packed_data: 1\n" - "packed_data: 4\n" - "packed_data: 1\n" - "packed_data: 1\n" - "packed_data: 7\n"; - -// A 3x20 Matrix of expected values for the result of the matrix multiply -// computed using R. -// Each column of this matrix is an expected output. -const char kExpectedText[] = - "rows: 3\n" - "cols: 20\n" - "packed_data: 12499\n" - "packed_data: 26793\n" - "packed_data: 16967\n" - "packed_data: 5007\n" - "packed_data: 14406\n" - "packed_data: 9635\n" - "packed_data: 4179\n" - "packed_data: 7870\n" - "packed_data: 4434\n" - "packed_data: 5793\n" - "packed_data: 12045\n" - "packed_data: 8876\n" - "packed_data: 2801\n" - "packed_data: 8053\n" - "packed_data: 5611\n" - "packed_data: 1740\n" - "packed_data: 4469\n" - "packed_data: 2665\n" - "packed_data: 8108\n" - "packed_data: 18396\n" - "packed_data: 10186\n" - "packed_data: 12330\n" - "packed_data: 23374\n" - "packed_data: 15526\n" - "packed_data: 9611\n" - "packed_data: 21804\n" - "packed_data: 14776\n" - "packed_data: 8241\n" - "packed_data: 17979\n" - "packed_data: 11989\n" - "packed_data: 8429\n" - "packed_data: 18921\n" - "packed_data: 9819\n" - "packed_data: 6270\n" - "packed_data: 13689\n" - "packed_data: 7031\n" - "packed_data: 9472\n" - "packed_data: 19210\n" - "packed_data: 13634\n" - "packed_data: 8567\n" - "packed_data: 12499\n" - "packed_data: 10455\n" - "packed_data: 2151\n" - "packed_data: 7469\n" - "packed_data: 3195\n" - "packed_data: 10774\n" - "packed_data: 21851\n" - "packed_data: 12673\n" - "packed_data: 12516\n" - "packed_data: 25318\n" - "packed_data: 14347\n" - "packed_data: 7984\n" - "packed_data: 17100\n" - "packed_data: 10972\n" - "packed_data: 5195\n" - "packed_data: 11102\n" - "packed_data: 8710\n" - "packed_data: 3002\n" - "packed_data: 11295\n" - "packed_data: 6360\n"; - -// Send a number of samples through the MatrixMultiplyCalculator. -TEST(MatrixMultiplyCalculatorTest, Multiply) { - CalculatorRunner runner("MatrixMultiplyCalculator", "", 1, 1, 1); - Matrix* matrix = new Matrix(); - MatrixFromTextProto(kMatrixText, matrix); - runner.MutableSidePackets()->Index(0) = Adopt(matrix); - - Matrix samples; - MatrixFromTextProto(kSamplesText, &samples); - Matrix expected; - MatrixFromTextProto(kExpectedText, &expected); - CHECK_EQ(samples.cols(), expected.cols()); - - for (int i = 0; i < samples.cols(); ++i) { - // Take a column from samples and produce a packet with just that - // column in it as an input sample for the calculator. - Eigen::MatrixXf* sample = new Eigen::MatrixXf(samples.block(0, i, 4, 1)); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(sample).At(Timestamp(i))); - } - - MP_ASSERT_OK(runner.Run()); - EXPECT_EQ(runner.MutableInputs()->Index(0).packets.size(), - runner.Outputs().Index(0).packets.size()); - - int i = 0; - for (const Packet& output : runner.Outputs().Index(0).packets) { - EXPECT_EQ(Timestamp(i), output.Timestamp()); - const Eigen::MatrixXf& result = output.Get(); - ASSERT_EQ(3, result.rows()); - EXPECT_NEAR((expected.block(0, i, 3, 1) - result).cwiseAbs().sum(), 0.0, - 1e-5); - ++i; - } - EXPECT_EQ(samples.cols(), i); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_subtract_calculator.cc b/mediapipe/calculators/core/matrix_subtract_calculator.cc deleted file mode 100644 index 09471a5ee..000000000 --- a/mediapipe/calculators/core/matrix_subtract_calculator.cc +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "Eigen/Core" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace api2 { - -// Subtract input matrix from the side input matrix and vice versa. The matrices -// must have the same dimension. -// Based on the tag (MINUEND vs SUBTRAHEND), the matrices in the input stream -// and input side packet can be either subtrahend or minuend. The output matrix -// is generated by performing minuend matrix - subtrahend matrix. -// -// Example config: -// node { -// calculator: "MatrixSubtractCalculator" -// input_stream: "MINUEND:input_matrix" -// input_side_packet: "SUBTRAHEND:side_matrix" -// output_stream: "output_matrix" -// } -// -// or -// -// node { -// calculator: "MatrixSubtractCalculator" -// input_stream: "SUBTRAHEND:input_matrix" -// input_side_packet: "MINUEND:side_matrix" -// output_stream: "output_matrix" -// } -class MatrixSubtractCalculator : public Node { - public: - static constexpr Input::SideFallback kMinuend{"MINUEND"}; - static constexpr Input::SideFallback kSubtrahend{"SUBTRAHEND"}; - static constexpr Output kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kMinuend, kSubtrahend, kOut); - static absl::Status UpdateContract(CalculatorContract* cc); - - absl::Status Process(CalculatorContext* cc) override; -}; -MEDIAPIPE_REGISTER_NODE(MatrixSubtractCalculator); - -// static -absl::Status MatrixSubtractCalculator::UpdateContract(CalculatorContract* cc) { - // TODO: the next restriction could be relaxed. - RET_CHECK(kMinuend(cc).IsStream() ^ kSubtrahend(cc).IsStream()) - << "MatrixSubtractCalculator only accepts exactly one input stream and " - "one input side packet"; - return absl::OkStatus(); -} - -absl::Status MatrixSubtractCalculator::Process(CalculatorContext* cc) { - const Matrix& minuend = *kMinuend(cc); - const Matrix& subtrahend = *kSubtrahend(cc); - if (minuend.rows() != subtrahend.rows() || - minuend.cols() != subtrahend.cols()) { - return absl::InvalidArgumentError( - "Minuend and subtrahend must have the same dimensions."); - } - kOut(cc).Send(minuend - subtrahend); - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_subtract_calculator_test.cc b/mediapipe/calculators/core/matrix_subtract_calculator_test.cc deleted file mode 100644 index 0bbf94dc8..000000000 --- a/mediapipe/calculators/core/matrix_subtract_calculator_test.cc +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "Eigen/Core" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { -namespace { - -// A 3x4 Matrix of random integers in [0,1000). -const char kMatrixText[] = - "rows: 3\n" - "cols: 4\n" - "packed_data: 387\n" - "packed_data: 940\n" - "packed_data: 815\n" - "packed_data: 825\n" - "packed_data: 997\n" - "packed_data: 884\n" - "packed_data: 419\n" - "packed_data: 763\n" - "packed_data: 123\n" - "packed_data: 30\n" - "packed_data: 825\n" - "packed_data: 299\n"; - -const char kMatrixText2[] = - "rows: 3\n" - "cols: 4\n" - "packed_data: 388\n" - "packed_data: 941\n" - "packed_data: 816\n" - "packed_data: 826\n" - "packed_data: 998\n" - "packed_data: 885\n" - "packed_data: 420\n" - "packed_data: 764\n" - "packed_data: 124\n" - "packed_data: 31\n" - "packed_data: 826\n" - "packed_data: 300\n"; - -TEST(MatrixSubtractCalculatorTest, WrongConfig) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "MatrixSubtractCalculator" - input_stream: "input_matrix" - input_side_packet: "SUBTRAHEND:side_matrix" - input_side_packet: "MINUEND:side_matrix2" - output_stream: "output_matrix" - )pb"); - CalculatorRunner runner(node_config); - auto status = runner.Run(); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "only accepts exactly one input stream and one input side packet")); -} - -TEST(MatrixSubtractCalculatorTest, WrongConfig2) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "MatrixSubtractCalculator" - input_side_packet: "SUBTRAHEND:side_matrix" - input_stream: "SUBTRAHEND:side_matrix2" - output_stream: "output_matrix" - )pb"); - CalculatorRunner runner(node_config); - auto status = runner.Run(); - EXPECT_THAT(status.message(), testing::HasSubstr("must be connected")); - EXPECT_THAT(status.message(), testing::HasSubstr("not both")); -} - -TEST(MatrixSubtractCalculatorTest, SubtractFromInput) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "MatrixSubtractCalculator" - input_stream: "MINUEND:input_matrix" - input_side_packet: "SUBTRAHEND:side_matrix" - output_stream: "output_matrix" - )pb"); - CalculatorRunner runner(node_config); - Matrix* side_matrix = new Matrix(); - MatrixFromTextProto(kMatrixText, side_matrix); - runner.MutableSidePackets()->Tag("SUBTRAHEND") = Adopt(side_matrix); - - Matrix* input_matrix = new Matrix(); - MatrixFromTextProto(kMatrixText2, input_matrix); - runner.MutableInputs()->Tag("MINUEND").packets.push_back( - Adopt(input_matrix).At(Timestamp(0))); - - MP_ASSERT_OK(runner.Run()); - EXPECT_EQ(1, runner.Outputs().Index(0).packets.size()); - - EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp()); - const Eigen::MatrixXf& result = - runner.Outputs().Index(0).packets[0].Get(); - ASSERT_EQ(3, result.rows()); - ASSERT_EQ(4, result.cols()); - EXPECT_NEAR(result.sum(), 12, 1e-5); -} - -TEST(MatrixSubtractCalculatorTest, SubtractFromSideMatrix) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "MatrixSubtractCalculator" - input_stream: "SUBTRAHEND:input_matrix" - input_side_packet: "MINUEND:side_matrix" - output_stream: "output_matrix" - )pb"); - CalculatorRunner runner(node_config); - Matrix* side_matrix = new Matrix(); - MatrixFromTextProto(kMatrixText, side_matrix); - runner.MutableSidePackets()->Tag("MINUEND") = Adopt(side_matrix); - - Matrix* input_matrix = new Matrix(); - MatrixFromTextProto(kMatrixText2, input_matrix); - runner.MutableInputs() - ->Tag("SUBTRAHEND") - .packets.push_back(Adopt(input_matrix).At(Timestamp(0))); - - MP_ASSERT_OK(runner.Run()); - EXPECT_EQ(1, runner.Outputs().Index(0).packets.size()); - - EXPECT_EQ(Timestamp(0), runner.Outputs().Index(0).packets[0].Timestamp()); - const Eigen::MatrixXf& result = - runner.Outputs().Index(0).packets[0].Get(); - ASSERT_EQ(3, result.rows()); - ASSERT_EQ(4, result.cols()); - EXPECT_NEAR(result.sum(), -12, 1e-5); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_to_vector_calculator.cc b/mediapipe/calculators/core/matrix_to_vector_calculator.cc deleted file mode 100644 index 90a36053b..000000000 --- a/mediapipe/calculators/core/matrix_to_vector_calculator.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Defines MatrixToVectorCalculator. -#include - -#include -#include -#include - -#include "Eigen/Core" -#include "absl/memory/memory.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/tool/status_util.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { -namespace api2 { - -// A calculator that converts a Matrix M to a vector containing all the -// entries of M in column-major order. -// -// Example config: -// node { -// calculator: "MatrixToVectorCalculator" -// input_stream: "input_matrix" -// output_stream: "column_major_vector" -// } -class MatrixToVectorCalculator : public Node { - public: - static constexpr Input kIn{""}; - static constexpr Output> kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut); - - absl::Status Open(CalculatorContext* cc) override; - - // Outputs a packet containing a vector for each input packet. - absl::Status Process(CalculatorContext* cc) override; -}; -MEDIAPIPE_REGISTER_NODE(MatrixToVectorCalculator); - -absl::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(0); - return mediapipe::OkStatus(); -} - -absl::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) { - const Matrix& input = *kIn(cc); - auto output = absl::make_unique>(); - - // The following lines work to convert the Matrix to a vector because Matrix - // is an Eigen::MatrixXf and Eigen uses column-major layout by default. - output->resize(input.rows() * input.cols()); - auto output_as_matrix = - Eigen::Map(output->data(), input.rows(), input.cols()); - output_as_matrix = input; - - kOut(cc).Send(std::move(output)); - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc b/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc deleted file mode 100644 index 1f994cbed..000000000 --- a/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "mediapipe/util/time_series_test_util.h" -#include "mediapipe/util/time_series_util.h" - -namespace mediapipe { -namespace { - -class MatrixToVectorCalculatorTest - : public mediapipe::TimeSeriesCalculatorTest { - protected: - void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; } - - void AppendInput(const std::vector& column_major_data, - int64 timestamp) { - ASSERT_EQ(num_input_samples_ * num_input_channels_, - column_major_data.size()); - Eigen::Map data_map(&column_major_data[0], - num_input_channels_, num_input_samples_); - AppendInputPacket(new Matrix(data_map), timestamp); - } - - void SetInputStreamParameters(int num_channels, int num_samples) { - num_input_channels_ = num_channels; - num_input_samples_ = num_samples; - input_sample_rate_ = 100; - input_packet_rate_ = 20.0; - } - - void SetInputHeader(int num_channels, int num_samples) { - SetInputStreamParameters(num_channels, num_samples); - FillInputHeader(); - } - - void CheckOutputPacket(int packet, std::vector expected_vector) { - const auto& actual_vector = - runner_->Outputs().Index(0).packets[packet].Get>(); - EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector)); - } -}; - -TEST_F(MatrixToVectorCalculatorTest, SingleRow) { - InitializeGraph(); - SetInputHeader(1, 4); // 1 channel x 4 samples - const std::vector& data_vector = {1.0, 2.0, 3.0, 4.0}; - AppendInput(data_vector, 0); - MP_ASSERT_OK(RunGraph()); - CheckOutputPacket(0, data_vector); -} - -TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) { - InitializeGraph(); - SetInputHeader(4, 2); // 4 channels x 2 samples - // Actual data matrix is the transpose of the appearance below. - const std::vector& data_vector = {1.0, 2.0, 3.0, 4.0, - 5.0, 6.0, 7.0, 8.0}; - AppendInput(data_vector, 0); - - MP_ASSERT_OK(RunGraph()); - CheckOutputPacket(0, data_vector); -} - -} // namespace - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/merge_calculator.cc b/mediapipe/calculators/core/merge_calculator.cc deleted file mode 100644 index a283842ae..000000000 --- a/mediapipe/calculators/core/merge_calculator.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace api2 { - -// This calculator takes a set of input streams and combines them into a single -// output stream. The packets from different streams do not need to contain the -// same type. If there are packets arriving at the same time from two or more -// input streams, the packet corresponding to the input stream with the smallest -// index is passed to the output and the rest are ignored. -// -// Example use-case: -// Suppose we have two (or more) different algorithms for detecting shot -// boundaries and we need to merge their packets into a single stream. The -// algorithms may emit shot boundaries at the same time and their output types -// may not be compatible. Subsequent calculators that process the merged stream -// may be interested only in the timestamps of the shot boundary packets and so -// it may not even need to inspect the values stored inside the packets. -// -// Example config: -// node { -// calculator: "MergeCalculator" -// input_stream: "shot_info1" -// input_stream: "shot_info2" -// input_stream: "shot_info3" -// output_stream: "merged_shot_infos" -// } -// -class MergeCalculator : public Node { - public: - static constexpr Input::Multiple kIn{""}; - static constexpr Output kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream"; - if (kIn(cc).Count() == 1) { - LOG(WARNING) - << "MergeCalculator expects multiple input streams to merge but is " - "receiving only one. Make sure the calculator is configured " - "correctly or consider removing this calculator to reduce " - "unnecessary overhead."; - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - // Output the packet from the first input stream with a packet ready at this - // timestamp. - for (const auto& input : kIn(cc)) { - if (!input.IsEmpty()) { - kOut(cc).Send(input.packet()); - return absl::OkStatus(); - } - } - - LOG(WARNING) << "Empty input packets at timestamp " - << cc->InputTimestamp().Value(); - - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(MergeCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/merge_calculator_test.cc b/mediapipe/calculators/core/merge_calculator_test.cc deleted file mode 100644 index 42170e1ea..000000000 --- a/mediapipe/calculators/core/merge_calculator_test.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -// Checks that the calculator fails if no input streams are provided. -TEST(InvariantMergeInputStreamsCalculator, NoInputStreamsMustFail) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "MergeCalculator" - output_stream: "merged_output" - )pb")); - // Expect calculator to fail. - ASSERT_FALSE(runner.Run().ok()); -} - -// Checks that the calculator fails with an incorrect number of output streams. -TEST(InvariantMergeInputStreamsCalculator, ExpectExactlyOneOutputStream) { - CalculatorRunner runner1( - ParseTextProtoOrDie(R"pb( - calculator: "MergeCalculator" - input_stream: "input1" - input_stream: "input2" - )pb")); - // Expect calculator to fail. - EXPECT_FALSE(runner1.Run().ok()); - - CalculatorRunner runner2( - ParseTextProtoOrDie(R"pb( - calculator: "MergeCalculator" - input_stream: "input1" - input_stream: "input2" - output_stream: "output1" - output_stream: "output2" - )pb")); - // Expect calculator to fail. - ASSERT_FALSE(runner2.Run().ok()); -} - -// Ensures two streams with differing types can be merged correctly. -TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest, - TestMergingTwoStreams) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "MergeCalculator" - input_stream: "input1" - input_stream: "input2" - output_stream: "combined_output" - )pb")); - - // input1: integers 10, 20, 30, occurring at times 10, 20, 30. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int(10)).At(Timestamp(10))); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int(20)).At(Timestamp(20))); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int(30)).At(Timestamp(30))); - // input2: floats 5.5, 35.5 at times 5, 35. - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(new float(5.5)).At(Timestamp(5))); - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(new float(35.5)).At(Timestamp(35))); - - MP_ASSERT_OK(runner.Run()); - - // Expected combined_output: 5.5, 10, 20, 30, 35.5 at times 5, 10, 20, 30, 35. - const std::vector& actual_output = runner.Outputs().Index(0).packets; - ASSERT_EQ(actual_output.size(), 5); - EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(5)); - EXPECT_EQ(actual_output[0].Get(), 5.5); - - EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(10)); - EXPECT_EQ(actual_output[1].Get(), 10); - - EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(20)); - EXPECT_EQ(actual_output[2].Get(), 20); - - EXPECT_EQ(actual_output[3].Timestamp(), Timestamp(30)); - EXPECT_EQ(actual_output[3].Get(), 30); - - EXPECT_EQ(actual_output[4].Timestamp(), Timestamp(35)); - EXPECT_EQ(actual_output[4].Get(), 35.5); -} - -// Ensures three streams with differing types can be merged correctly. -TEST(MediaPipeDetectionToSoapboxDetectionCalculatorTest, - TestMergingThreeStreams) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "MergeCalculator" - input_stream: "input1" - input_stream: "input2" - input_stream: "input3" - output_stream: "combined_output" - )pb")); - - // input1: integer 30 occurring at time 30. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int(30)).At(Timestamp(30))); - // input2: float 20.5 occurring at time 20. - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(new float(20.5)).At(Timestamp(20))); - // input3: char 'c' occurring at time 10. - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(new char('c')).At(Timestamp(10))); - - MP_ASSERT_OK(runner.Run()); - - // Expected combined_output: 'c', 20.5, 30 at times 10, 20, 30. - const std::vector& actual_output = runner.Outputs().Index(0).packets; - ASSERT_EQ(actual_output.size(), 3); - EXPECT_EQ(actual_output[0].Timestamp(), Timestamp(10)); - EXPECT_EQ(actual_output[0].Get(), 'c'); - - EXPECT_EQ(actual_output[1].Timestamp(), Timestamp(20)); - EXPECT_EQ(actual_output[1].Get(), 20.5); - - EXPECT_EQ(actual_output[2].Timestamp(), Timestamp(30)); - EXPECT_EQ(actual_output[2].Get(), 30); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/mux_calculator.cc b/mediapipe/calculators/core/mux_calculator.cc deleted file mode 100644 index a0ce2ae34..000000000 --- a/mediapipe/calculators/core/mux_calculator.cc +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace api2 { - -// A Calculator that selects an input stream from "INPUT:0", "INPUT:1", ..., -// using the integer value (0, 1, ...) in the packet on the "SELECT" input -// stream, and passes the packet on the selected input stream to the "OUTPUT" -// output stream. -// -// Note that this calculator defaults to use MuxInputStreamHandler, which is -// required for this calculator. However, it can be overridden to work with -// other InputStreamHandlers. Check out the unit tests on for an example usage -// with DefaultInputStreamHandler. -// TODO: why would you need to use DefaultISH? Perhaps b/167596925? -class MuxCalculator : public Node { - public: - static constexpr Input::SideFallback kSelect{"SELECT"}; - // TODO: this currently sets them all to Any independently, instead - // of the first being Any and the others being SameAs. - static constexpr Input::Multiple kIn{"INPUT"}; - static constexpr Output> kOut{"OUTPUT"}; - - MEDIAPIPE_NODE_CONTRACT(kSelect, kIn, kOut, - StreamHandler("MuxInputStreamHandler")); - - absl::Status Process(CalculatorContext* cc) final { - int select = *kSelect(cc); - RET_CHECK(0 <= select && select < kIn(cc).Count()); - if (!kIn(cc)[select].IsEmpty()) { - kOut(cc).Send(kIn(cc)[select].packet()); - } - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(MuxCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/mux_calculator_test.cc b/mediapipe/calculators/core/mux_calculator_test.cc deleted file mode 100644 index e2dd74553..000000000 --- a/mediapipe/calculators/core/mux_calculator_test.cc +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/split_vector_calculator.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -typedef SplitVectorCalculator SplitIntVectorCalculator; -REGISTER_CALCULATOR(SplitIntVectorCalculator); - -namespace { - -// Graph with default input stream handler, and the input selection is driven -// by an input stream. All MuxCalculator inputs are present at each timestamp. -constexpr char kTestGraphConfig1[] = R"pb( - input_stream: "input" - output_stream: "test_output" - node { - calculator: "SplitIntVectorCalculator" - input_stream: "input" - output_stream: "stream0" - output_stream: "stream1" - output_stream: "stream2" - output_stream: "input_select" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 2 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 3 end: 4 } - element_only: true - } - } - } - node { - calculator: "MuxCalculator" - input_stream: "INPUT:0:stream0" - input_stream: "INPUT:1:stream1" - input_stream: "INPUT:2:stream2" - input_stream: "SELECT:input_select" - output_stream: "OUTPUT:test_output" - input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } - } -)pb"; - -// Graph with default input stream handler, and the input selection is driven -// by an input side packet. All MuxCalculator inputs are present at each -// timestamp. -constexpr char kTestGraphConfig2[] = R"pb( - input_side_packet: "input_selector" - input_stream: "input" - output_stream: "test_output" - node { - calculator: "SplitIntVectorCalculator" - input_stream: "input" - output_stream: "stream0" - output_stream: "stream1" - output_stream: "stream2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 2 } - ranges: { begin: 2 end: 3 } - element_only: true - } - } - } - node { - calculator: "MuxCalculator" - input_stream: "INPUT:0:stream0" - input_stream: "INPUT:1:stream1" - input_stream: "INPUT:2:stream2" - input_side_packet: "SELECT:input_selector" - output_stream: "OUTPUT:test_output" - input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } - } -)pb"; - -// Graph with mux input stream handler, and the input selection is driven -// by an input stream. Only one MuxCalculator input is present at each -// timestamp. -constexpr char kTestGraphConfig3[] = R"pb( - input_stream: "input" - output_stream: "test_output" - node { - calculator: "RoundRobinDemuxCalculator" - input_stream: "input" - output_stream: "OUTPUT:0:stream0" - output_stream: "OUTPUT:1:stream1" - output_stream: "OUTPUT:2:stream2" - output_stream: "SELECT:input_select" - } - node { - calculator: "MuxCalculator" - input_stream: "INPUT:0:stream0" - input_stream: "INPUT:1:stream1" - input_stream: "INPUT:2:stream2" - input_stream: "SELECT:input_select" - output_stream: "OUTPUT:test_output" - } -)pb"; - -constexpr char kOutputName[] = "test_output"; -constexpr char kInputName[] = "input"; -constexpr char kInputSelector[] = "input_selector"; - -// Helper to run a graph with the given inputs and generate outputs, asserting -// each step along the way. -// Inputs: -// graph_config_proto - graph config protobuf -// extra_side_packets - input side packets name to value map -// input_stream_name - name of the input -void RunGraph(const std::string& graph_config_proto, - const std::map& extra_side_packets, - const std::string& input_stream_name, int num_input_packets, - std::function input_fn, - const std::string& output_stream_name, - std::function output_fn) { - CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie(graph_config_proto); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - MP_ASSERT_OK(graph.ObserveOutputStream(output_stream_name, output_fn)); - MP_ASSERT_OK(graph.StartRun(extra_side_packets)); - for (int i = 0; i < num_input_packets; ++i) { - MP_ASSERT_OK(graph.AddPacketToInputStream(input_stream_name, input_fn(i))); - } - MP_ASSERT_OK(graph.CloseAllInputStreams()); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST(MuxCalculatorTest, InputStreamSelector_DefaultInputStreamHandler) { - // Input and handling. - std::vector> input_packets = { - {1, 1, 2, 1}, {3, 5, 8, 2}, {13, 21, 34, 0}, - {55, 89, 144, 2}, {233, 377, 610, 0}, {987, 1597, 2584, 1}, - {4181, 6765, 10946, 2}, - }; - int packet_time_stamp = 22; - // This function will return the i-th input packet. - auto input_fn = [&packet_time_stamp, &input_packets](int i) -> Packet { - return MakePacket>(input_packets[i]) - .At(Timestamp(packet_time_stamp++)); - }; - - // Output and handling. - std::vector output; - // This function collects the output from the packet. - auto output_fn = [&output](const Packet& p) -> absl::Status { - output.push_back(p.Get()); - return absl::OkStatus(); - }; - - RunGraph(kTestGraphConfig1, {}, kInputName, input_packets.size(), input_fn, - kOutputName, output_fn); - EXPECT_THAT(output, testing::ElementsAre(1, 8, 13, 144, 233, 1597, 10946)); -} - -TEST(MuxCalculatorTest, InputSidePacketSelector_DefaultInputStreamHandler) { - // Input and handling. - std::vector> input_packets = { - {1, 1, 2}, {3, 5, 8}, {13, 21, 34}, {55, 89, 144}, - {233, 377, 610}, {987, 1597, 2584}, {4181, 6765, 10946}, - }; - int packet_time_stamp = 22; - // This function will return the i-th input packet. - auto input_fn = [&packet_time_stamp, &input_packets](int i) -> Packet { - return MakePacket>(input_packets[i]) - .At(Timestamp(packet_time_stamp++)); - }; - - // Output and handling. - std::vector output; - // This function collects the output from the packet. - auto output_fn = [&output](const Packet& p) -> absl::Status { - output.push_back(p.Get()); - return absl::OkStatus(); - }; - - RunGraph(kTestGraphConfig2, {{kInputSelector, MakePacket(0)}}, - kInputName, input_packets.size(), input_fn, kOutputName, output_fn); - EXPECT_THAT(output, testing::ElementsAre(1, 3, 13, 55, 233, 987, 4181)); - - output.clear(); - RunGraph(kTestGraphConfig2, {{kInputSelector, MakePacket(1)}}, - kInputName, input_packets.size(), input_fn, kOutputName, output_fn); - EXPECT_THAT(output, testing::ElementsAre(1, 5, 21, 89, 377, 1597, 6765)); - - output.clear(); - RunGraph(kTestGraphConfig2, {{kInputSelector, MakePacket(2)}}, - kInputName, input_packets.size(), input_fn, kOutputName, output_fn); - EXPECT_THAT(output, testing::ElementsAre(2, 8, 34, 144, 610, 2584, 10946)); -} - -TEST(MuxCalculatorTest, InputStreamSelector_MuxInputStreamHandler) { - // Input and handling. - std::vector input_packets = {1, 1, 2, 3, 5, 8, 13, - 21, 34, 55, 89, 144, 233, 377, - 610, 987, 1597, 2584, 4181, 6765, 10946}; - int packet_time_stamp = 22; - // This function will return the i-th input packet. - auto input_fn = [&packet_time_stamp, &input_packets](int i) -> Packet { - return MakePacket(input_packets[i]).At(Timestamp(packet_time_stamp++)); - }; - - // Output and handling. - std::vector output; - // This function collects the output from the packet. - auto output_fn = [&output](const Packet& p) -> absl::Status { - output.push_back(p.Get()); - return absl::OkStatus(); - }; - - RunGraph(kTestGraphConfig3, {}, kInputName, input_packets.size(), input_fn, - kOutputName, output_fn); - EXPECT_EQ(output, input_packets); -} - -constexpr char kDualInputGraphConfig[] = R"pb( - input_stream: "input_0" - input_stream: "input_1" - input_stream: "input_select" - output_stream: "test_output" - node { - calculator: "MuxCalculator" - input_stream: "INPUT:0:input_0" - input_stream: "INPUT:1:input_1" - input_stream: "SELECT:input_select" - output_stream: "OUTPUT:test_output" - } -)pb"; - -TEST(MuxCalculatorTest, DiscardSkippedInputs_MuxInputStreamHandler) { - CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - kDualInputGraphConfig); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - - std::shared_ptr output; - MP_ASSERT_OK( - graph.ObserveOutputStream("test_output", [&output](const Packet& p) { - output = p.Get>(); - return absl::OkStatus(); - })); - - MP_ASSERT_OK(graph.StartRun({})); - - auto one = std::make_shared(1); - auto two = std::make_shared(2); - auto three = std::make_shared(3); - std::weak_ptr one_weak = one; - std::weak_ptr two_weak = two; - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_0", - MakePacket>(std::move(one)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_1", - MakePacket>(std::move(two)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_1", - MakePacket>(std::move(three)).At(Timestamp(1)))); - EXPECT_EQ(one, nullptr); - EXPECT_EQ(two, nullptr); - EXPECT_EQ(three, nullptr); - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_select", MakePacket(0).At(Timestamp(0)))); - MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_EQ(*output, 1); - EXPECT_NE(one_weak.lock(), nullptr); - EXPECT_EQ(two_weak.lock(), nullptr); - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_select", MakePacket(1).At(Timestamp(1)))); - MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_EQ(*output, 3); - - MP_ASSERT_OK(graph.CloseAllInputStreams()); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/non_zero_calculator.cc b/mediapipe/calculators/core/non_zero_calculator.cc deleted file mode 100644 index 6555fbf2f..000000000 --- a/mediapipe/calculators/core/non_zero_calculator.cc +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace api2 { - -// A Calculator that returns 0 if INPUT is 0, and 1 otherwise. -class NonZeroCalculator : public Node { - public: - static constexpr Input::SideFallback kIn{"INPUT"}; - static constexpr Output::Optional kOut{"OUTPUT"}; - static constexpr Output::Optional kBooleanOut{"OUTPUT_BOOL"}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut, kBooleanOut); - - absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK(kOut(cc).IsConnected() || kBooleanOut(cc).IsConnected()) - << "At least one output stream is expected."; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - if (!kIn(cc).IsEmpty()) { - bool isNonZero = *kIn(cc) != 0; - if (kOut(cc).IsConnected()) { - kOut(cc).Send(std::make_unique(isNonZero ? 1 : 0)); - } - if (kBooleanOut(cc).IsConnected()) { - kBooleanOut(cc).Send(std::make_unique(isNonZero)); - } - } - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(NonZeroCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/non_zero_calculator_test.cc b/mediapipe/calculators/core/non_zero_calculator_test.cc deleted file mode 100644 index 72f5fa1d7..000000000 --- a/mediapipe/calculators/core/non_zero_calculator_test.cc +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -class NonZeroCalculatorTest : public ::testing::Test { - protected: - NonZeroCalculatorTest() - : runner_( - R"pb( - calculator: "NonZeroCalculator" - input_stream: "INPUT:input" - output_stream: "OUTPUT:output" - output_stream: "OUTPUT_BOOL:output_bool" - )pb") {} - - void SetInput(const std::vector& inputs) { - int timestamp = 0; - for (const auto input : inputs) { - runner_.MutableInputs() - ->Get("INPUT", 0) - .packets.push_back(MakePacket(input).At(Timestamp(timestamp++))); - } - } - - std::vector GetOutput() { - std::vector result; - for (const auto output : runner_.Outputs().Get("OUTPUT", 0).packets) { - result.push_back(output.Get()); - } - return result; - } - - std::vector GetOutputBool() { - std::vector result; - for (const auto output : runner_.Outputs().Get("OUTPUT_BOOL", 0).packets) { - result.push_back(output.Get()); - } - return result; - } - - CalculatorRunner runner_; -}; - -TEST_F(NonZeroCalculatorTest, ProducesZeroOutputForZeroInput) { - SetInput({0}); - - MP_ASSERT_OK(runner_.Run()); - - EXPECT_THAT(GetOutput(), ::testing::ElementsAre(0)); - EXPECT_THAT(GetOutputBool(), ::testing::ElementsAre(false)); -} - -TEST_F(NonZeroCalculatorTest, ProducesNonZeroOutputForNonZeroInput) { - SetInput({1, 2, 3, -4, 5}); - - MP_ASSERT_OK(runner_.Run()); - - EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 1, 1, 1, 1)); - EXPECT_THAT(GetOutputBool(), - ::testing::ElementsAre(true, true, true, true, true)); -} - -TEST_F(NonZeroCalculatorTest, SwitchesBetweenNonZeroAndZeroOutput) { - SetInput({1, 0, 3, 0, 5}); - MP_ASSERT_OK(runner_.Run()); - EXPECT_THAT(GetOutput(), ::testing::ElementsAre(1, 0, 1, 0, 1)); - EXPECT_THAT(GetOutputBool(), - ::testing::ElementsAre(true, false, true, false, true)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_cloner_calculator.cc b/mediapipe/calculators/core/packet_cloner_calculator.cc deleted file mode 100644 index 41bddbfa7..000000000 --- a/mediapipe/calculators/core/packet_cloner_calculator.cc +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// This takes packets from N+1 streams, A_1, A_2, ..., A_N, B. -// For every packet that appears in B, outputs the most recent packet from each -// of the A_i on a separate stream. - -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/core/packet_cloner_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" - -namespace mediapipe { - -// For every packet received on the last stream, output the latest packet -// obtained on all other streams. Therefore, if the last stream outputs at a -// higher rate than the others, this effectively clones the packets from the -// other streams to match the last. -// -// Example config: -// node { -// calculator: "PacketClonerCalculator" -// input_stream: "first_base_signal" -// input_stream: "second_base_signal" -// input_stream: "tick_signal" -// output_stream: "cloned_first_base_signal" -// output_stream: "cloned_second_base_signal" -// } -// -// Related: -// packet_cloner_calculator.proto: Options for this calculator. -// merge_input_streams_calculator.cc: One output stream. -// packet_inner_join_calculator.cc: Don't output unless all inputs are new. -class PacketClonerCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const int tick_signal_index = cc->Inputs().NumEntries() - 1; - for (int i = 0; i < tick_signal_index; ++i) { - cc->Inputs().Index(i).SetAny(); - cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i)); - } - cc->Inputs().Index(tick_signal_index).SetAny(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - // Load options. - const auto calculator_options = - cc->Options(); - output_only_when_all_inputs_received_ = - calculator_options.output_only_when_all_inputs_received(); - - // Parse input streams. - tick_signal_index_ = cc->Inputs().NumEntries() - 1; - current_.resize(tick_signal_index_); - // Pass along the header for each stream if present. - for (int i = 0; i < tick_signal_index_; ++i) { - if (!cc->Inputs().Index(i).Header().IsEmpty()) { - cc->Outputs().Index(i).SetHeader(cc->Inputs().Index(i).Header()); - } - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - // Store input signals. - for (int i = 0; i < tick_signal_index_; ++i) { - if (!cc->Inputs().Index(i).Value().IsEmpty()) { - current_[i] = cc->Inputs().Index(i).Value(); - } - } - - // Output according to the TICK signal. - if (!cc->Inputs().Index(tick_signal_index_).Value().IsEmpty()) { - if (output_only_when_all_inputs_received_) { - // Return if one of the input is null. - for (int i = 0; i < tick_signal_index_; ++i) { - if (current_[i].IsEmpty()) { - return absl::OkStatus(); - } - } - } - // Output each stream. - for (int i = 0; i < tick_signal_index_; ++i) { - if (!current_[i].IsEmpty()) { - cc->Outputs().Index(i).AddPacket( - current_[i].At(cc->InputTimestamp())); - } else { - cc->Outputs().Index(i).SetNextTimestampBound( - cc->InputTimestamp().NextAllowedInStream()); - } - } - } - return absl::OkStatus(); - } - - private: - std::vector current_; - int tick_signal_index_; - bool output_only_when_all_inputs_received_; -}; - -REGISTER_CALCULATOR(PacketClonerCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_cloner_calculator.proto b/mediapipe/calculators/core/packet_cloner_calculator.proto deleted file mode 100644 index e30672fab..000000000 --- a/mediapipe/calculators/core/packet_cloner_calculator.proto +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message PacketClonerCalculatorOptions { - extend CalculatorOptions { - optional PacketClonerCalculatorOptions ext = 258872085; - } - - // When true, this calculator will drop received TICK packets if any input - // stream hasn't received a packet yet. - optional bool output_only_when_all_inputs_received = 1 [default = false]; -} diff --git a/mediapipe/calculators/core/packet_inner_join_calculator.cc b/mediapipe/calculators/core/packet_inner_join_calculator.cc deleted file mode 100644 index 6ffffb58b..000000000 --- a/mediapipe/calculators/core/packet_inner_join_calculator.cc +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -// Calculator that acts like the SQL query: -// SELECT * -// FROM packets_on_stream1 AS packet1 -// INNER JOIN packets_on_stream2 AS packet2 -// ON packet1.timestamp = packet2.timestamp -// -// In other words, it only emits and forwards packets if all input streams are -// not empty. -// -// Intended for use with FixedSizeInputStreamHandler. -// -// Related: -// packet_cloner_calculator.cc: Repeats last-seen packets from empty inputs. -class PacketInnerJoinCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - int num_streams_; -}; - -REGISTER_CALCULATOR(PacketInnerJoinCalculator); - -absl::Status PacketInnerJoinCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().NumEntries() == cc->Outputs().NumEntries()) - << "The number of input and output streams must match."; - const int num_streams = cc->Inputs().NumEntries(); - for (int i = 0; i < num_streams; ++i) { - cc->Inputs().Index(i).SetAny(); - cc->Outputs().Index(i).SetSameAs(&cc->Inputs().Index(i)); - } - return absl::OkStatus(); -} - -absl::Status PacketInnerJoinCalculator::Open(CalculatorContext* cc) { - num_streams_ = cc->Inputs().NumEntries(); - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status PacketInnerJoinCalculator::Process(CalculatorContext* cc) { - for (int i = 0; i < num_streams_; ++i) { - if (cc->Inputs().Index(i).Value().IsEmpty()) { - return absl::OkStatus(); - } - } - for (int i = 0; i < num_streams_; ++i) { - cc->Outputs().Index(i).AddPacket(cc->Inputs().Index(i).Value()); - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_inner_join_calculator_test.cc b/mediapipe/calculators/core/packet_inner_join_calculator_test.cc deleted file mode 100644 index 4f4d9884c..000000000 --- a/mediapipe/calculators/core/packet_inner_join_calculator_test.cc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { -namespace { - -inline Packet PacketFrom(int i) { return Adopt(new int(i)).At(Timestamp(i)); } - -TEST(PacketInnerJoinCalculatorTest, AllMatching) { - // Test case. - const std::vector packets_on_stream1 = {0, 1, 2, 3}; - const std::vector packets_on_stream2 = {0, 1, 2, 3}; - // Run. - CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0); - for (int packet_load : packets_on_stream1) { - runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load)); - } - for (int packet_load : packets_on_stream2) { - runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load)); - } - MP_ASSERT_OK(runner.Run()); - // Check. - const std::vector expected = {0, 1, 2, 3}; - ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size()); - ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size()); - for (int i = 0; i < expected.size(); ++i) { - const Packet packet1 = runner.Outputs().Index(0).packets[i]; - EXPECT_EQ(expected[i], packet1.Get()); - EXPECT_EQ(expected[i], packet1.Timestamp().Value()); - const Packet packet2 = runner.Outputs().Index(1).packets[i]; - EXPECT_EQ(expected[i], packet2.Get()); - EXPECT_EQ(expected[i], packet2.Timestamp().Value()); - } -} - -TEST(PacketInnerJoinCalculatorTest, NoneMatching) { - // Test case. - const std::vector packets_on_stream1 = {0, 2}; - const std::vector packets_on_stream2 = {1, 3}; - // Run. - CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0); - for (int packet_load : packets_on_stream1) { - runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load)); - } - for (int packet_load : packets_on_stream2) { - runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load)); - } - MP_ASSERT_OK(runner.Run()); - // Check. - EXPECT_TRUE(runner.Outputs().Index(0).packets.empty()); - EXPECT_TRUE(runner.Outputs().Index(1).packets.empty()); -} - -TEST(PacketInnerJoinCalculatorTest, SomeMatching) { - // Test case. - const std::vector packets_on_stream1 = {0, 1, 2, 3, 4, 6}; - const std::vector packets_on_stream2 = {0, 2, 4, 5, 6}; - // Run. - CalculatorRunner runner("PacketInnerJoinCalculator", "", 2, 2, 0); - for (int packet_load : packets_on_stream1) { - runner.MutableInputs()->Index(0).packets.push_back(PacketFrom(packet_load)); - } - for (int packet_load : packets_on_stream2) { - runner.MutableInputs()->Index(1).packets.push_back(PacketFrom(packet_load)); - } - MP_ASSERT_OK(runner.Run()); - // Check. - const std::vector expected = {0, 2, 4, 6}; - ASSERT_EQ(expected.size(), runner.Outputs().Index(0).packets.size()); - ASSERT_EQ(expected.size(), runner.Outputs().Index(1).packets.size()); - for (int i = 0; i < expected.size(); ++i) { - const Packet packet1 = runner.Outputs().Index(0).packets[i]; - EXPECT_EQ(expected[i], packet1.Get()); - EXPECT_EQ(expected[i], packet1.Timestamp().Value()); - const Packet packet2 = runner.Outputs().Index(1).packets[i]; - EXPECT_EQ(expected[i], packet2.Get()); - EXPECT_EQ(expected[i], packet2.Timestamp().Value()); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_presence_calculator.cc b/mediapipe/calculators/core/packet_presence_calculator.cc deleted file mode 100644 index cb119a76d..000000000 --- a/mediapipe/calculators/core/packet_presence_calculator.cc +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// For each non empty input packet, emits a single output packet containing a -// boolean value "true", "false" in response to empty packets (a.k.a. timestamp -// bound updates) This can be used to "flag" the presence of an arbitrary packet -// type as input into a downstream calculator. -// -// Inputs: -// PACKET - any type. -// -// Outputs: -// PRESENCE - bool. -// "true" if packet is not empty, "false" if there's timestamp bound update -// instead. -// -// Examples: -// node: { -// calculator: "PacketPresenceCalculator" -// input_stream: "PACKET:packet" -// output_stream: "PRESENCE:presence" -// } -// -// This calculator can be used in conjuction with GateCalculator in order to -// allow/disallow processing. For instance: -// node: { -// calculator: "PacketPresenceCalculator" -// input_stream: "PACKET:value" -// output_stream: "PRESENCE:disallow_if_present" -// } -// node { -// calculator: "GateCalculator" -// input_stream: "image" -// input_stream: "DISALLOW:disallow_if_present" -// output_stream: "image_for_processing" -// options: { -// [mediapipe.GateCalculatorOptions.ext] { -// empty_packets_as_allow: true -// } -// } -// } -class PacketPresenceCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag("PACKET").SetAny(); - cc->Outputs().Tag("PRESENCE").Set(); - // Process() function is invoked in response to input stream timestamp - // bound updates. - cc->SetProcessTimestampBounds(true); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - cc->Outputs() - .Tag("PRESENCE") - .AddPacket(MakePacket(!cc->Inputs().Tag("PACKET").IsEmpty()) - .At(cc->InputTimestamp())); - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(PacketPresenceCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_presence_calculator_test.cc b/mediapipe/calculators/core/packet_presence_calculator_test.cc deleted file mode 100644 index 54185d8ac..000000000 --- a/mediapipe/calculators/core/packet_presence_calculator_test.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/sink.h" - -namespace mediapipe { -using ::testing::ElementsAre; -using ::testing::Eq; -using ::testing::Value; -namespace { - -MATCHER_P2(BoolPacket, value, timestamp, "") { - return Value(arg.template Get(), Eq(value)) && - Value(arg.Timestamp(), Eq(timestamp)); -} - -TEST(PreviousLoopbackCalculator, CorrectTimestamps) { - std::vector output_packets; - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'allow' - input_stream: 'value' - node { - calculator: "GateCalculator" - input_stream: 'value' - input_stream: 'ALLOW:allow' - output_stream: 'gated_value' - } - node { - calculator: 'PacketPresenceCalculator' - input_stream: 'PACKET:gated_value' - output_stream: 'PRESENCE:presence' - } - )pb"); - tool::AddVectorSink("presence", &graph_config, &output_packets); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config, {})); - MP_ASSERT_OK(graph.StartRun({})); - - auto send_packet = [&graph](int value, bool allow, Timestamp timestamp) { - MP_ASSERT_OK(graph.AddPacketToInputStream( - "value", MakePacket(value).At(timestamp))); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "allow", MakePacket(allow).At(timestamp))); - }; - - send_packet(10, false, Timestamp(10)); - MP_EXPECT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets, ElementsAre(BoolPacket(false, Timestamp(10)))); - - output_packets.clear(); - send_packet(20, true, Timestamp(11)); - MP_EXPECT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets, ElementsAre(BoolPacket(true, Timestamp(11)))); - - MP_EXPECT_OK(graph.CloseAllInputStreams()); - MP_EXPECT_OK(graph.WaitUntilDone()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_resampler_calculator.cc b/mediapipe/calculators/core/packet_resampler_calculator.cc deleted file mode 100644 index 43253520a..000000000 --- a/mediapipe/calculators/core/packet_resampler_calculator.cc +++ /dev/null @@ -1,696 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/packet_resampler_calculator.h" - -#include - -namespace { -// Reflect an integer against the lower and upper bound of an interval. -int64 ReflectBetween(int64 ts, int64 ts_min, int64 ts_max) { - if (ts < ts_min) return 2 * ts_min - ts - 1; - if (ts >= ts_max) return 2 * ts_max - ts - 1; - return ts; -} - -// Creates a secure random number generator for use in ProcessWithJitter. -// If no secure random number generator can be constructed, the jitter -// option is disabled in order to mainatain a consistent security and -// consistent random seeding. -std::unique_ptr CreateSecureRandom(const std::string& seed) { - RandomBase* result = nullptr; - return std::unique_ptr(result); -} - -} // namespace - -namespace mediapipe { - -REGISTER_CALCULATOR(PacketResamplerCalculator); -namespace { -// Returns a TimestampDiff (assuming microseconds) corresponding to the -// given time in seconds. -TimestampDiff TimestampDiffFromSeconds(double seconds) { - return TimestampDiff(MathUtil::SafeRound( - seconds * Timestamp::kTimestampUnitsPerSecond)); -} -} // namespace - -absl::Status PacketResamplerCalculator::GetContract(CalculatorContract* cc) { - const auto& resampler_options = - cc->Options(); - if (cc->InputSidePackets().HasTag("OPTIONS")) { - cc->InputSidePackets().Tag("OPTIONS").Set(); - } - CollectionItemId input_data_id = cc->Inputs().GetId("DATA", 0); - if (!input_data_id.IsValid()) { - input_data_id = cc->Inputs().GetId("", 0); - } - cc->Inputs().Get(input_data_id).SetAny(); - if (cc->Inputs().HasTag("VIDEO_HEADER")) { - cc->Inputs().Tag("VIDEO_HEADER").Set(); - } - - CollectionItemId output_data_id = cc->Outputs().GetId("DATA", 0); - if (!output_data_id.IsValid()) { - output_data_id = cc->Outputs().GetId("", 0); - } - cc->Outputs().Get(output_data_id).SetSameAs(&cc->Inputs().Get(input_data_id)); - if (cc->Outputs().HasTag("VIDEO_HEADER")) { - cc->Outputs().Tag("VIDEO_HEADER").Set(); - } - - if (resampler_options.jitter() != 0.0) { - RET_CHECK_GT(resampler_options.jitter(), 0.0); - RET_CHECK_LE(resampler_options.jitter(), 1.0); - RET_CHECK(cc->InputSidePackets().HasTag("SEED")); - cc->InputSidePackets().Tag("SEED").Set(); - } - return absl::OkStatus(); -} - -absl::Status PacketResamplerCalculator::Open(CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - - flush_last_packet_ = resampler_options.flush_last_packet(); - jitter_ = resampler_options.jitter(); - - input_data_id_ = cc->Inputs().GetId("DATA", 0); - if (!input_data_id_.IsValid()) { - input_data_id_ = cc->Inputs().GetId("", 0); - } - output_data_id_ = cc->Outputs().GetId("DATA", 0); - if (!output_data_id_.IsValid()) { - output_data_id_ = cc->Outputs().GetId("", 0); - } - - frame_rate_ = resampler_options.frame_rate(); - start_time_ = resampler_options.has_start_time() - ? Timestamp(resampler_options.start_time()) - : Timestamp::Min(); - end_time_ = resampler_options.has_end_time() - ? Timestamp(resampler_options.end_time()) - : Timestamp::Max(); - round_limits_ = resampler_options.round_limits(); - // The frame_rate has a default value of -1.0, so the user must set it! - RET_CHECK_LT(0, frame_rate_) - << "The output frame rate must be greater than zero"; - RET_CHECK_LE(frame_rate_, Timestamp::kTimestampUnitsPerSecond) - << "The output frame rate must be smaller than " - << Timestamp::kTimestampUnitsPerSecond; - - frame_time_usec_ = static_cast(1000000.0 / frame_rate_); - jitter_usec_ = static_cast(1000000.0 * jitter_ / frame_rate_); - RET_CHECK_LE(jitter_usec_, frame_time_usec_); - - video_header_.frame_rate = frame_rate_; - - if (resampler_options.output_header() != - PacketResamplerCalculatorOptions::NONE && - !cc->Inputs().Get(input_data_id_).Header().IsEmpty()) { - if (resampler_options.output_header() == - PacketResamplerCalculatorOptions::UPDATE_VIDEO_HEADER) { - video_header_ = - cc->Inputs().Get(input_data_id_).Header().Get(); - video_header_.frame_rate = frame_rate_; - cc->Outputs() - .Get(output_data_id_) - .SetHeader(Adopt(new VideoHeader(video_header_))); - } else { - cc->Outputs() - .Get(output_data_id_) - .SetHeader(cc->Inputs().Get(input_data_id_).Header()); - } - } - - strategy_ = GetSamplingStrategy(resampler_options); - - return strategy_->Open(cc); -} - -absl::Status PacketResamplerCalculator::Process(CalculatorContext* cc) { - if (cc->InputTimestamp() == Timestamp::PreStream() && - cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") && - !cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) { - video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get(); - video_header_.frame_rate = frame_rate_; - if (cc->Inputs().Get(input_data_id_).IsEmpty()) { - return absl::OkStatus(); - } - } - - if (absl::Status status = strategy_->Process(cc); !status.ok()) { - return status; // Avoid MP_RETURN_IF_ERROR macro for external release. - } - - last_packet_ = cc->Inputs().Get(input_data_id_).Value(); - - return absl::OkStatus(); -} - -absl::Status PacketResamplerCalculator::Close(CalculatorContext* cc) { - if (!cc->GraphStatus().ok()) { - return absl::OkStatus(); - } - - return strategy_->Close(cc); -} - -std::unique_ptr -PacketResamplerCalculator::GetSamplingStrategy( - const PacketResamplerCalculatorOptions& options) { - if (options.reproducible_sampling()) { - if (!options.jitter_with_reflection()) { - LOG(WARNING) - << "reproducible_sampling enabled w/ jitter_with_reflection " - "disabled. " - << "reproducible_sampling always uses jitter with reflection, " - << "Ignoring jitter_with_reflection setting."; - } - return absl::make_unique(this); - } - - if (options.jitter() == 0) { - return absl::make_unique(this); - } - - if (options.jitter_with_reflection()) { - return absl::make_unique(this); - } - - // With jitter and no reflection. - return absl::make_unique(this); -} - -Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(int64 index) const { - CHECK_EQ(jitter_, 0.0); - CHECK_NE(first_timestamp_, Timestamp::Unset()); - return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_); -} - -int64 PacketResamplerCalculator::TimestampToPeriodIndex( - Timestamp timestamp) const { - CHECK_EQ(jitter_, 0.0); - CHECK_NE(first_timestamp_, Timestamp::Unset()); - return MathUtil::SafeRound( - (timestamp - first_timestamp_).Seconds() * frame_rate_); -} - -void PacketResamplerCalculator::OutputWithinLimits(CalculatorContext* cc, - const Packet& packet) const { - TimestampDiff margin((round_limits_) ? frame_time_usec_ / 2 : 0); - if (packet.Timestamp() >= start_time_ - margin && - packet.Timestamp() < end_time_ + margin) { - cc->Outputs().Get(output_data_id_).AddPacket(packet); - } -} - -absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - - if (resampler_options.output_header() != - PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; - } - - if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; - } - - const auto& seed = cc->InputSidePackets().Tag("SEED").Get(); - random_ = CreateSecureRandom(seed); - if (random_ == nullptr) { - return absl::InvalidArgumentError( - "SecureRandom is not available. With \"jitter\" specified, " - "PacketResamplerCalculator processing cannot proceed."); - } - - packet_reservoir_random_ = CreateSecureRandom(seed); - packet_reservoir_ = - std::make_unique(packet_reservoir_random_.get()); - - return absl::OkStatus(); -} -absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) { - if (!packet_reservoir_->IsEmpty()) { - LOG(INFO) << "Emitting pack from reservoir."; - calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample()); - } - return absl::OkStatus(); -} -absl::Status LegacyJitterWithReflectionStrategy::Process( - CalculatorContext* cc) { - RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream()); - - if (packet_reservoir_->IsEnabled() && - (first_timestamp_ == Timestamp::Unset() || - (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) { - auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value(); - packet_reservoir_->AddSample(curr_packet); - } - - if (first_timestamp_ == Timestamp::Unset()) { - first_timestamp_ = cc->InputTimestamp(); - InitializeNextOutputTimestampWithJitter(); - if (first_timestamp_ == next_output_timestamp_) { - calculator_->OutputWithinLimits(cc, cc->Inputs() - .Get(calculator_->input_data_id_) - .Value() - .At(next_output_timestamp_)); - UpdateNextOutputTimestampWithJitter(); - } - return absl::OkStatus(); - } - - if (calculator_->frame_time_usec_ < - (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { - LOG_FIRST_N(WARNING, 2) - << "Adding jitter is not very useful when upsampling."; - } - - while (true) { - const int64 last_diff = - (next_output_timestamp_ - calculator_->last_packet_.Timestamp()) - .Value(); - RET_CHECK_GT(last_diff, 0); - const int64 curr_diff = - (next_output_timestamp_ - cc->InputTimestamp()).Value(); - if (curr_diff > 0) { - break; - } - calculator_->OutputWithinLimits( - cc, (std::abs(curr_diff) > last_diff - ? calculator_->last_packet_ - : cc->Inputs().Get(calculator_->input_data_id_).Value()) - .At(next_output_timestamp_)); - UpdateNextOutputTimestampWithJitter(); - // From now on every time a packet is emitted the timestamp of the next - // packet becomes known; that timestamp is stored in next_output_timestamp_. - // The only exception to this rule is the packet emitted from Close() which - // can only happen when jitter_with_reflection is enabled but in this case - // next_output_timestamp_min_ is a non-decreasing lower bound of any - // subsequent packet. - const Timestamp timestamp_bound = next_output_timestamp_min_; - cc->Outputs() - .Get(calculator_->output_data_id_) - .SetNextTimestampBound(timestamp_bound); - } - return absl::OkStatus(); -} - -void LegacyJitterWithReflectionStrategy:: - InitializeNextOutputTimestampWithJitter() { - next_output_timestamp_min_ = first_timestamp_; - next_output_timestamp_ = - first_timestamp_ + - random_->UnbiasedUniform64(calculator_->frame_time_usec_); -} - -void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() { - packet_reservoir_->Clear(); - next_output_timestamp_min_ += calculator_->frame_time_usec_; - Timestamp next_output_timestamp_max_ = - next_output_timestamp_min_ + calculator_->frame_time_usec_; - - next_output_timestamp_ += - calculator_->frame_time_usec_ + - random_->UnbiasedUniform64(2 * calculator_->jitter_usec_ + 1) - - calculator_->jitter_usec_; - next_output_timestamp_ = Timestamp(ReflectBetween( - next_output_timestamp_.Value(), next_output_timestamp_min_.Value(), - next_output_timestamp_max_.Value())); - CHECK_GE(next_output_timestamp_, next_output_timestamp_min_); - CHECK_LT(next_output_timestamp_, next_output_timestamp_max_); -} - -absl::Status ReproducibleJitterWithReflectionStrategy::Open( - CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - - if (resampler_options.output_header() != - PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; - } - - if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; - } - - const auto& seed = cc->InputSidePackets().Tag("SEED").Get(); - random_ = CreateSecureRandom(seed); - if (random_ == nullptr) { - return absl::InvalidArgumentError( - "SecureRandom is not available. With \"jitter\" specified, " - "PacketResamplerCalculator processing cannot proceed."); - } - - return absl::OkStatus(); -} -absl::Status ReproducibleJitterWithReflectionStrategy::Close( - CalculatorContext* cc) { - // If last packet is non-empty and a packet hasn't been emitted for this - // period, emit the last packet. - if (!calculator_->last_packet_.IsEmpty() && !packet_emitted_this_period_) { - calculator_->OutputWithinLimits( - cc, calculator_->last_packet_.At(next_output_timestamp_)); - } - return absl::OkStatus(); -} -absl::Status ReproducibleJitterWithReflectionStrategy::Process( - CalculatorContext* cc) { - RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream()); - - Packet current_packet = cc->Inputs().Get(calculator_->input_data_id_).Value(); - - if (calculator_->last_packet_.IsEmpty()) { - // last_packet is empty, this is the first packet of the stream. - - InitializeNextOutputTimestamp(current_packet.Timestamp()); - - // If next_output_timestamp_ happens to fall before current_packet, emit - // current packet. Only a single packet can be emitted at the beginning - // of the stream. - if (next_output_timestamp_ < current_packet.Timestamp()) { - calculator_->OutputWithinLimits( - cc, current_packet.At(next_output_timestamp_)); - packet_emitted_this_period_ = true; - } - - return absl::OkStatus(); - } - - // Last packet is set, so we are mid-stream. - if (calculator_->frame_time_usec_ < - (current_packet.Timestamp() - calculator_->last_packet_.Timestamp()) - .Value()) { - // Note, if the stream is upsampling, this could lead to the same packet - // being emitted twice. Upsampling and jitter doesn't make much sense - // but does technically work. - LOG_FIRST_N(WARNING, 2) - << "Adding jitter is not very useful when upsampling."; - } - - // Since we may be upsampling, we need to iteratively advance the - // next_output_timestamp_ one period at a time until it reaches the period - // current_packet is in. During this process, last_packet and/or - // current_packet may be repeatly emitted. - - UpdateNextOutputTimestamp(current_packet.Timestamp()); - - while (!packet_emitted_this_period_ && - next_output_timestamp_ <= current_packet.Timestamp()) { - // last_packet < next_output_timestamp_ <= current_packet, - // so emit the closest packet. - Packet packet_to_emit = - current_packet.Timestamp() - next_output_timestamp_ < - next_output_timestamp_ - calculator_->last_packet_.Timestamp() - ? current_packet - : calculator_->last_packet_; - calculator_->OutputWithinLimits(cc, - packet_to_emit.At(next_output_timestamp_)); - - packet_emitted_this_period_ = true; - - // If we are upsampling, packet_emitted_this_period_ can be reset by - // the following UpdateNext and the loop will iterate. - UpdateNextOutputTimestamp(current_packet.Timestamp()); - } - - // Set the bounds on the output stream. Note, if we emitted a packet - // above, it will already be set at next_output_timestamp_ + 1, in which - // case we have to skip setting it. - if (cc->Outputs().Get(calculator_->output_data_id_).NextTimestampBound() < - next_output_timestamp_) { - cc->Outputs() - .Get(calculator_->output_data_id_) - .SetNextTimestampBound(next_output_timestamp_); - } - return absl::OkStatus(); -} - -void ReproducibleJitterWithReflectionStrategy::InitializeNextOutputTimestamp( - Timestamp current_timestamp) { - if (next_output_timestamp_min_ != Timestamp::Unset()) { - return; - } - - next_output_timestamp_min_ = Timestamp(0); - next_output_timestamp_ = - Timestamp(GetNextRandom(calculator_->frame_time_usec_)); - - // While the current timestamp is ahead of the max (i.e. min + frame_time), - // fast-forward. - while (current_timestamp >= - next_output_timestamp_min_ + calculator_->frame_time_usec_) { - packet_emitted_this_period_ = true; // Force update... - UpdateNextOutputTimestamp(current_timestamp); - } -} - -void ReproducibleJitterWithReflectionStrategy::UpdateNextOutputTimestamp( - Timestamp current_timestamp) { - if (packet_emitted_this_period_ && - current_timestamp >= - next_output_timestamp_min_ + calculator_->frame_time_usec_) { - next_output_timestamp_min_ += calculator_->frame_time_usec_; - Timestamp next_output_timestamp_max_ = - next_output_timestamp_min_ + calculator_->frame_time_usec_; - - next_output_timestamp_ += calculator_->frame_time_usec_ + - GetNextRandom(2 * calculator_->jitter_usec_ + 1) - - calculator_->jitter_usec_; - next_output_timestamp_ = Timestamp(ReflectBetween( - next_output_timestamp_.Value(), next_output_timestamp_min_.Value(), - next_output_timestamp_max_.Value())); - - packet_emitted_this_period_ = false; - } -} - -absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - - if (resampler_options.output_header() != - PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; - } - - if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; - } - - const auto& seed = cc->InputSidePackets().Tag("SEED").Get(); - random_ = CreateSecureRandom(seed); - if (random_ == nullptr) { - return absl::InvalidArgumentError( - "SecureRandom is not available. With \"jitter\" specified, " - "PacketResamplerCalculator processing cannot proceed."); - } - - packet_reservoir_random_ = CreateSecureRandom(seed); - packet_reservoir_ = - absl::make_unique(packet_reservoir_random_.get()); - - return absl::OkStatus(); -} -absl::Status JitterWithoutReflectionStrategy::Close(CalculatorContext* cc) { - if (!packet_reservoir_->IsEmpty()) { - calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample()); - } - return absl::OkStatus(); -} -absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) { - RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream()); - - // Packet reservior is used to make sure there's an output for every period, - // e.g. partial period at the end of the stream. - if (packet_reservoir_->IsEnabled() && - (calculator_->first_timestamp_ == Timestamp::Unset() || - (cc->InputTimestamp() - next_output_timestamp_min_).Value() >= 0)) { - auto curr_packet = cc->Inputs().Get(calculator_->input_data_id_).Value(); - packet_reservoir_->AddSample(curr_packet); - } - - if (calculator_->first_timestamp_ == Timestamp::Unset()) { - calculator_->first_timestamp_ = cc->InputTimestamp(); - InitializeNextOutputTimestamp(); - if (calculator_->first_timestamp_ == next_output_timestamp_) { - calculator_->OutputWithinLimits(cc, cc->Inputs() - .Get(calculator_->input_data_id_) - .Value() - .At(next_output_timestamp_)); - UpdateNextOutputTimestamp(); - } - return absl::OkStatus(); - } - - if (calculator_->frame_time_usec_ < - (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { - LOG_FIRST_N(WARNING, 2) - << "Adding jitter is not very useful when upsampling."; - } - - while (true) { - const int64 last_diff = - (next_output_timestamp_ - calculator_->last_packet_.Timestamp()) - .Value(); - RET_CHECK_GT(last_diff, 0); - const int64 curr_diff = - (next_output_timestamp_ - cc->InputTimestamp()).Value(); - if (curr_diff > 0) { - break; - } - calculator_->OutputWithinLimits( - cc, (std::abs(curr_diff) > last_diff - ? calculator_->last_packet_ - : cc->Inputs().Get(calculator_->input_data_id_).Value()) - .At(next_output_timestamp_)); - UpdateNextOutputTimestamp(); - cc->Outputs() - .Get(calculator_->output_data_id_) - .SetNextTimestampBound(next_output_timestamp_); - } - return absl::OkStatus(); -} - -void JitterWithoutReflectionStrategy::InitializeNextOutputTimestamp() { - next_output_timestamp_min_ = calculator_->first_timestamp_; - next_output_timestamp_ = calculator_->first_timestamp_ + - calculator_->frame_time_usec_ * random_->RandFloat(); -} - -void JitterWithoutReflectionStrategy::UpdateNextOutputTimestamp() { - packet_reservoir_->Clear(); - packet_reservoir_->Disable(); - next_output_timestamp_ += calculator_->frame_time_usec_ * - ((1.0 - calculator_->jitter_) + - 2.0 * calculator_->jitter_ * random_->RandFloat()); -} - -absl::Status NoJitterStrategy::Open(CalculatorContext* cc) { - const auto resampler_options = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), "OPTIONS"); - base_timestamp_ = resampler_options.has_base_timestamp() - ? Timestamp(resampler_options.base_timestamp()) - : Timestamp::Unset(); - - period_count_ = 0; - - return absl::OkStatus(); -} -absl::Status NoJitterStrategy::Close(CalculatorContext* cc) { - // Emit the last packet received if we have at least one packet, but - // haven't sent anything for its period. - if (calculator_->first_timestamp_ != Timestamp::Unset() && - calculator_->flush_last_packet_ && - calculator_->TimestampToPeriodIndex( - calculator_->last_packet_.Timestamp()) == period_count_) { - calculator_->OutputWithinLimits( - cc, calculator_->last_packet_.At( - calculator_->PeriodIndexToTimestamp(period_count_))); - } - return absl::OkStatus(); -} -absl::Status NoJitterStrategy::Process(CalculatorContext* cc) { - RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream()); - - if (calculator_->first_timestamp_ == Timestamp::Unset()) { - // This is the first packet, initialize the first_timestamp_. - if (base_timestamp_ == Timestamp::Unset()) { - // Initialize first_timestamp_ with exactly the first packet timestamp. - calculator_->first_timestamp_ = cc->InputTimestamp(); - } else { - // Initialize first_timestamp_ with the first packet timestamp - // aligned to the base_timestamp_. - int64 first_index = MathUtil::SafeRound( - (cc->InputTimestamp() - base_timestamp_).Seconds() * - calculator_->frame_rate_); - calculator_->first_timestamp_ = - base_timestamp_ + - TimestampDiffFromSeconds(first_index / calculator_->frame_rate_); - } - if (cc->Outputs().UsesTags() && cc->Outputs().HasTag("VIDEO_HEADER")) { - cc->Outputs() - .Tag("VIDEO_HEADER") - .Add(new VideoHeader(calculator_->video_header_), - Timestamp::PreStream()); - } - } - const Timestamp received_timestamp = cc->InputTimestamp(); - const int64 received_timestamp_idx = - calculator_->TimestampToPeriodIndex(received_timestamp); - // Only consider the received packet if it belongs to the current period - // (== period_count_) or to a newer one (> period_count_). - if (received_timestamp_idx >= period_count_) { - // Fill the empty periods until we are in the same index as the received - // packet. - while (received_timestamp_idx > period_count_) { - calculator_->OutputWithinLimits( - cc, calculator_->last_packet_.At( - calculator_->PeriodIndexToTimestamp(period_count_))); - ++period_count_; - } - // Now, if the received packet has a timestamp larger than the middle of - // the current period, we can send a packet without waiting. We send the - // one closer to the middle. - Timestamp target_timestamp = - calculator_->PeriodIndexToTimestamp(period_count_); - if (received_timestamp >= target_timestamp) { - bool have_last_packet = - (calculator_->last_packet_.Timestamp() != Timestamp::Unset()); - bool send_current = - !have_last_packet || - (received_timestamp - target_timestamp <= - target_timestamp - calculator_->last_packet_.Timestamp()); - if (send_current) { - calculator_->OutputWithinLimits(cc, - cc->Inputs() - .Get(calculator_->input_data_id_) - .Value() - .At(target_timestamp)); - } else { - calculator_->OutputWithinLimits( - cc, calculator_->last_packet_.At(target_timestamp)); - } - ++period_count_; - } - // TODO: Add a mechanism to the framework to allow these packets - // to be output earlier (without waiting for a much later packet to - // arrive) - - // Update the bound for the next packet. - cc->Outputs() - .Get(calculator_->output_data_id_) - .SetNextTimestampBound( - calculator_->PeriodIndexToTimestamp(period_count_)); - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_resampler_calculator.h b/mediapipe/calculators/core/packet_resampler_calculator.h deleted file mode 100644 index fbecdb0e7..000000000 --- a/mediapipe/calculators/core/packet_resampler_calculator.h +++ /dev/null @@ -1,392 +0,0 @@ -#ifndef MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_ - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/deps/mathutil.h" -#include "mediapipe/framework/deps/random_base.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/framework/tool/options_util.h" - -namespace mediapipe { - -class PacketReservoir { - public: - PacketReservoir(RandomBase* rng) : rng_(rng) {} - // Replace candidate with current packet with 1/count_ probability. - void AddSample(Packet sample) { - if (rng_->UnbiasedUniform(++count_) == 0) { - reservoir_ = sample; - } - } - bool IsEnabled() { return rng_ && enabled_; } - void Disable() { - if (enabled_) enabled_ = false; - } - void Clear() { count_ = 0; } - bool IsEmpty() { return count_ == 0; } - Packet GetSample() { return reservoir_; } - - private: - RandomBase* rng_; - bool enabled_ = true; - int32 count_ = 0; - Packet reservoir_; -}; - -// This calculator is used to normalize the frequency of the packets -// out of a stream. Given a desired frame rate, packets are going to be -// removed or added to achieve it. -// -// If jitter_ is specified: -// - The first packet is chosen randomly (uniform distribution) among frames -// that correspond to timestamps [0, 1/frame_rate). Let the chosen packet -// correspond to timestamp t. -// - The next packet is chosen randomly (uniform distribution) among frames -// that correspond to [t+(1-jitter)/frame_rate, t+(1+jitter)/frame_rate]. -// - if jitter_with_reflection is true, the timestamp will be reflected -// against the boundaries of [t_0 + (k-1)/frame_rate, t_0 + k/frame_rate) -// so that its marginal distribution is uniform within this interval. -// In the formula, t_0 is the timestamp of the first sampled -// packet, and the k is the packet index. -// See paper (https://arxiv.org/abs/2002.01147) for details. -// - t is updated and the process is repeated. -// - Note that seed is specified as input side packet for reproducibility of -// the resampling. For Cloud ML Video Intelligence API, the hash of the -// input video should serve this purpose. For YouTube, either video ID or -// content hex ID of the input video should do. -// - If reproducible_samping is true, care is taken to allow reproducible -// "mid-stream" sampling. The calculator can be executed on a stream that -// doesn't start at the first period. For instance, if the calculator -// is run on a 10 second stream it will produce the same set of samples -// as two runs of the calculator, the first with 3 seconds of input starting -// at time 0 and the second with 7 seconds of input starting at time +3s. -// - In order to guarantee the exact same samples, 1) the inputs must be -// aligned with the sampling period. For instance, if the sampling rate -// is 2 frames per second, streams should be aligned on 0.5 second -// boundaries, and 2) the stream must include at least one extra packet -// before and after the second aligned sampling period. -// -// If jitter_ is not specified: -// - The first packet defines the first_timestamp of the output stream, -// so it is always emitted. -// - If more packets are emitted, they will have timestamp equal to -// round(first_timestamp + k * period) , where k is a positive -// integer and the period is defined by the frame rate. -// Example: first_timestamp=0, fps=30, then the output stream -// will have timestamps: 0, 33333, 66667, 100000, etc... -// - The packets selected for the output stream are the ones closer -// to the exact middle point (33333.33, 66666.67 in our previous -// example). In case of ties, later packets are chosen. -// - 'Empty' periods happen when there are no packets for a long time -// (greater than a period). In this case, we send a copy of the last -// packet received before the empty period. -// The jitter feature is disabled by default. To enable it, you need to -// implement CreateSecureRandom(const std::string&). -// -// The data stream may be either specified as the only stream (by index) -// or as the stream with tag "DATA". -// -// The input and output streams may be accompanied by a VIDEO_HEADER -// stream. This stream includes a VideoHeader at Timestamp::PreStream(). -// The input VideoHeader on the VIDEO_HEADER stream will always be updated -// with the resampler frame rate no matter what the options value for -// output_header is before being output on the output VIDEO_HEADER stream. -// If the input VideoHeader is not available, then only the frame rate -// value will be set in the output. -// -// Related: -// packet_downsampler_calculator.cc: skips packets regardless of timestamps. -class PacketResamplerCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - // Given the current count of periods that have passed, this returns - // the next valid timestamp of the middle point of the next period: - // if count is 0, it returns the first_timestamp_. - // if count is 1, it returns the first_timestamp_ + period (corresponding - // to the first tick using exact fps) - // e.g. for frame_rate=30 and first_timestamp_=0: - // 0: 0 - // 1: 33333 - // 2: 66667 - // 3: 100000 - // - // Can only be used if jitter_ equals zero. - Timestamp PeriodIndexToTimestamp(int64 index) const; - - // Given a Timestamp, finds the closest sync Timestamp based on - // first_timestamp_ and the desired fps. - // - // Can only be used if jitter_ equals zero. - int64 TimestampToPeriodIndex(Timestamp timestamp) const; - - // Outputs a packet if it is in range (start_time_, end_time_). - void OutputWithinLimits(CalculatorContext* cc, const Packet& packet) const; - - protected: - // Returns Sampling Strategy to use. - // - // Virtual to allow injection of testing strategies. - virtual std::unique_ptr GetSamplingStrategy( - const mediapipe::PacketResamplerCalculatorOptions& options); - - private: - std::unique_ptr strategy_; - - // The timestamp of the first packet received. - Timestamp first_timestamp_; - - // Number of frames per second (desired output frequency). - double frame_rate_; - - // Inverse of frame_rate_. - int64 frame_time_usec_; - - VideoHeader video_header_; - // The "DATA" input stream. - CollectionItemId input_data_id_; - // The "DATA" output stream. - CollectionItemId output_data_id_; - - // Indicator whether to flush last packet even if its timestamp is greater - // than the final stream timestamp. - bool flush_last_packet_; - - double jitter_ = 0.0; - - int64 jitter_usec_; - - // The last packet that was received. - Packet last_packet_; - - // If specified, only outputs at/after start_time are included. - Timestamp start_time_; - - // If specified, only outputs before end_time are included. - Timestamp end_time_; - - // If set, the output timestamps nearest to start_time and end_time - // are included in the output, even if the nearest timestamp is not - // between start_time and end_time. - bool round_limits_; - - // Allow strategies access to all internal calculator state. - // - // The calculator and strategies are intimiately tied together so this should - // not break encapsulation. - friend class LegacyJitterWithReflectionStrategy; - friend class ReproducibleJitterWithReflectionStrategy; - friend class JitterWithoutReflectionStrategy; - friend class NoJitterStrategy; -}; - -// Abstract class encapsulating sampling stategy. -// -// These are used solely by PacketResamplerCalculator, but are exposed here -// to facilitate tests. -class PacketResamplerStrategy { - public: - PacketResamplerStrategy(PacketResamplerCalculator* calculator) - : calculator_(calculator) {} - virtual ~PacketResamplerStrategy() = default; - - // Delegate for CalculatorBase::Open. See CalculatorBase for relevant - // implementation considerations. - virtual absl::Status Open(CalculatorContext* cc) = 0; - // Delegate for CalculatorBase::Close. See CalculatorBase for relevant - // implementation considerations. - virtual absl::Status Close(CalculatorContext* cc) = 0; - // Delegate for CalculatorBase::Process. See CalculatorBase for relevant - // implementation considerations. - virtual absl::Status Process(CalculatorContext* cc) = 0; - - protected: - // Calculator running strategy. - PacketResamplerCalculator* calculator_; -}; - -// Strategy that applies Jitter with reflection based sampling. -// -// Used by PacketResamplerCalculator when both Jitter and reflection are -// enabled. -// -// This applies the legacy jitter with reflection which doesn't allow -// for reproducibility of sampling when starting mid-stream. This is maintained -// for backward compatibility. -class LegacyJitterWithReflectionStrategy : public PacketResamplerStrategy { - public: - LegacyJitterWithReflectionStrategy(PacketResamplerCalculator* calculator) - : PacketResamplerStrategy(calculator) {} - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - void InitializeNextOutputTimestampWithJitter(); - void UpdateNextOutputTimestampWithJitter(); - - // Jitter-related variables. - std::unique_ptr random_; - - // The timestamp of the first packet received. - Timestamp first_timestamp_; - - // Next packet to be emitted. Since packets may not align perfectly with - // next_output_timestamp_, the closest packet will be emitted. - Timestamp next_output_timestamp_; - - // Lower bound for next timestamp. - // - // next_output_timestamp_ will be kept within the interval - // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_) - Timestamp next_output_timestamp_min_ = Timestamp::Unset(); - - // packet reservior used for sampling random packet out of partial - // period when jitter is enabled - std::unique_ptr packet_reservoir_; - - // random number generator used in packet_reservior_. - std::unique_ptr packet_reservoir_random_; -}; - -// Strategy that applies reproducible jitter with reflection based sampling. -// -// Used by PacketResamplerCalculator when both Jitter and reflection are -// enabled. -class ReproducibleJitterWithReflectionStrategy - : public PacketResamplerStrategy { - public: - ReproducibleJitterWithReflectionStrategy( - PacketResamplerCalculator* calculator) - : PacketResamplerStrategy(calculator) {} - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - protected: - // Returns next random in range (0,n]. - // - // Exposed as virtual function for testing Jitter with reflection. - // This is the only way random_ is accessed. - virtual uint64 GetNextRandom(uint64 n) { - return random_->UnbiasedUniform64(n); - } - - private: - // Initializes Jitter with reflection. - // - // This will fast-forward to the period containing current_timestamp. - // next_output_timestamp_ is guarnateed to be current_timestamp's period - // and packet_emitted_this_period_ will be set to false. - void InitializeNextOutputTimestamp(Timestamp current_timestamp); - - // Potentially advances next_output_timestamp_ a single period. - // - // next_output_timestamp_ will only be advanced if packet_emitted_this_period_ - // is false. next_output_timestamp_ will never be advanced beyond - // current_timestamp's period. - // - // However, next_output_timestamp_ could fall before current_timestamp's - // period since only a single period can be advanced at a time. - void UpdateNextOutputTimestamp(Timestamp current_timestamp); - - // Jitter-related variables. - std::unique_ptr random_; - - // Next packet to be emitted. Since packets may not align perfectly with - // next_output_timestamp_, the closest packet will be emitted. - Timestamp next_output_timestamp_; - - // Lower bound for next timestamp. - // - // next_output_timestamp_ will be kept within the interval - // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_) - Timestamp next_output_timestamp_min_ = Timestamp::Unset(); - - // Indicates packet was emitted for current period (i.e. the period - // next_output_timestamp_ falls in. - bool packet_emitted_this_period_ = false; -}; - -// Strategy that applies Jitter without reflection based sampling. -// -// Used by PacketResamplerCalculator when Jitter is enabled and reflection is -// not enabled. -class JitterWithoutReflectionStrategy : public PacketResamplerStrategy { - public: - JitterWithoutReflectionStrategy(PacketResamplerCalculator* calculator) - : PacketResamplerStrategy(calculator) {} - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Calculates the first sampled timestamp that incorporates a jittering - // offset. - void InitializeNextOutputTimestamp(); - - // Calculates the next sampled timestamp that incorporates a jittering offset. - void UpdateNextOutputTimestamp(); - - // Jitter-related variables. - std::unique_ptr random_; - - // Next packet to be emitted. Since packets may not align perfectly with - // next_output_timestamp_, the closest packet will be emitted. - Timestamp next_output_timestamp_; - - // Lower bound for next timestamp. - // - // next_output_timestamp_ will be kept within the interval - // [next_output_timestamp_min_, next_output_timestamp_min_ + frame_time_usec_) - Timestamp next_output_timestamp_min_ = Timestamp::Unset(); - - // packet reservior used for sampling random packet out of partial period. - std::unique_ptr packet_reservoir_; - - // random number generator used in packet_reservior_. - std::unique_ptr packet_reservoir_random_; -}; - -// Strategy that applies sampling without any jitter. -// -// Used by PacketResamplerCalculator when jitter is not enabled. -class NoJitterStrategy : public PacketResamplerStrategy { - public: - NoJitterStrategy(PacketResamplerCalculator* calculator) - : PacketResamplerStrategy(calculator) {} - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Number of periods that have passed (= #packets sent to the output). - int64 period_count_; - - // If specified, output timestamps are aligned with base_timestamp. - // Otherwise, they are aligned with the first input timestamp. - Timestamp base_timestamp_; -}; - -} // namespace mediapipe -#endif // MEDIAPIPE_CALCULATORS_CORE_PACKET_RESAMPLER_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/packet_resampler_calculator.proto b/mediapipe/calculators/core/packet_resampler_calculator.proto deleted file mode 100644 index f7ca47023..000000000 --- a/mediapipe/calculators/core/packet_resampler_calculator.proto +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message PacketResamplerCalculatorOptions { - extend CalculatorOptions { - optional PacketResamplerCalculatorOptions ext = 95743844; - } - - // The output frame rate measured in frames per second. - // - // The closest packet in time in each period will be chosen. If there - // is no packet in the period then the most recent packet will be chosen - // (not the closest in time). - optional double frame_rate = 1 [default = -1.0]; - - enum OutputHeader { - // Do not output a header, even if the input contained one. - NONE = 0; - // Pass the header, if the input contained one. - PASS_HEADER = 1; - // Update the frame rate in the header, which must be of type VideoHeader. - UPDATE_VIDEO_HEADER = 2; - } - - // Whether and what kind of header to place on the output stream. - // Note, this is about the actual header, not the VIDEO_HEADER stream. - // If this option is set to UPDATE_VIDEO_HEADER then the header will - // also be parsed (updated) and passed along to the VIDEO_HEADER stream. - optional OutputHeader output_header = 2 [default = NONE]; - - // Flush last packet even if its timestamp is greater than the final stream - // timestamp. - optional bool flush_last_packet = 3 [default = true]; - - // Adds jitter to resampling if set, so that Google's sampling is not - // externally deterministic. - // - // When set, the randomizer will be initialized with a seed. Then, the first - // sample is chosen randomly (uniform distribution) among frames that - // correspond to timestamps [0, 1/frame_rate). Let the chosen frame - // correspond to timestamp t. The next frame is chosen randomly (uniform - // distribution) among frames that correspond to [t+(1-jitter)/frame_rate, - // t+(1+jitter)/frame_rate]. t is updated and the process is repeated. - // - // Valid values are in the range of [0.0, 1.0] with the default being 0.0 (no - // jitter). A typical value would be a value in the range of 0.1-0.25. - // - // Note that this does NOT guarantee the desired frame rate, but if the - // pseudo-random number generator does its job and the number of frames is - // sufficiently large, the average frame rate will be close to this value. - optional double jitter = 4; - - // Enables reflection when applying jitter. - // - // This option is ignored when reproducible_sampling is true, in which case - // reflection will be used. - // - // New use cases should use reproducible_sampling = true, as - // jitter_with_reflection is deprecated and will be removed at some point. - optional bool jitter_with_reflection = 9 [default = false]; - - // If set, enabled reproducible sampling, allowing frames to be sampled - // without regards to where the stream starts. See - // packet_resampler_calculator.h for details. - // - // This enables reflection (ignoring jitter_with_reflection setting). - optional bool reproducible_sampling = 10 [default = false]; - - // If specified, output timestamps are aligned with base_timestamp. - // Otherwise, they are aligned with the first input timestamp. - // - // In order to ensure that the outptut timestamps are reproducible, - // with round_limits = false, the bounds for input timestamps must include: - // [start_time - period / 2, end_time + period / 2], - // with round_limits = true, the bounds for input timestamps must include: - // [start_time - period, end_time + period], - // where period = 1 / frame_rate. - // - // For example, in PacketResamplerCalculatorOptions specify - // "start_time: 3000000", and in MediaDecoderOptions specify - // "start_time: 2999950". - optional int64 base_timestamp = 5; - - // If specified, only outputs at/after start_time are included. - optional int64 start_time = 6; - - // If specified, only outputs before end_time are included. - optional int64 end_time = 7; - - // If set, the output timestamps nearest to start_time and end_time - // are included in the output, even if the nearest timestamp is not - // between start_time and end_time. - optional bool round_limits = 8 [default = false]; -} diff --git a/mediapipe/calculators/core/packet_resampler_calculator_test.cc b/mediapipe/calculators/core/packet_resampler_calculator_test.cc deleted file mode 100644 index 191e1d842..000000000 --- a/mediapipe/calculators/core/packet_resampler_calculator_test.cc +++ /dev/null @@ -1,752 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/packet_resampler_calculator.h" - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -using ::testing::ElementsAre; -namespace { -// A simple version of CalculatorRunner with built-in convenience -// methods for setting inputs from a vector and checking outputs -// against expected outputs (both timestamps and contents). -class SimpleRunner : public CalculatorRunner { - public: - explicit SimpleRunner(const std::string& options_string) - : CalculatorRunner("PacketResamplerCalculator", options_string, 1, 1, 0) { - } - explicit SimpleRunner(const CalculatorGraphConfig::Node& node_config) - : CalculatorRunner(node_config) {} - - virtual ~SimpleRunner() {} - - void SetInput(const std::vector& timestamp_list) { - MutableInputs()->Index(0).packets.clear(); - for (const int64 ts : timestamp_list) { - MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string(absl::StrCat("Frame #", ts))) - .At(Timestamp(ts))); - } - } - - void SetVideoHeader(const double frame_rate) { - video_header_.width = static_count_; - video_header_.height = static_count_ * 10; - video_header_.frame_rate = frame_rate; - video_header_.duration = static_count_ * 100.0; - video_header_.format = static_cast( - static_count_ % ImageFormat::Format_ARRAYSIZE); - MutableInputs()->Index(0).header = Adopt(new VideoHeader(video_header_)); - ++static_count_; - } - - void CheckOutputTimestamps( - const std::vector& expected_frames, - const std::vector& expected_timestamps) const { - EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size()); - EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size()); - int count = 0; - for (const Packet& packet : Outputs().Index(0).packets) { - EXPECT_EQ(Timestamp(expected_timestamps[count]), packet.Timestamp()); - const std::string& packet_contents = packet.Get(); - EXPECT_EQ(std::string(absl::StrCat("Frame #", expected_frames[count])), - packet_contents); - ++count; - } - } - - void CheckVideoHeader(const double expected_frame_rate) const { - ASSERT_FALSE(Outputs().Index(0).header.IsEmpty()); - const VideoHeader& header = Outputs().Index(0).header.Get(); - const double frame_rate = header.frame_rate; - - EXPECT_EQ(video_header_.width, header.width); - EXPECT_EQ(video_header_.height, header.height); - EXPECT_DOUBLE_EQ(expected_frame_rate, frame_rate); - EXPECT_FLOAT_EQ(video_header_.duration, header.duration); - EXPECT_EQ(video_header_.format, header.format); - } - - private: - VideoHeader video_header_; - static int static_count_; -}; - -// Matcher for Packets with uint64 payload, comparing arg packet's -// timestamp and uint64 payload. -MATCHER_P2(PacketAtTimestamp, payload, timestamp, - absl::StrCat(negation ? "isn't" : "is", " a packet with payload ", - payload, " @ time ", timestamp)) { - if (timestamp != arg.Timestamp().Value()) { - *result_listener << "at incorrect timestamp = " << arg.Timestamp().Value(); - return false; - } - int64 actual_payload = arg.template Get(); - if (actual_payload != payload) { - *result_listener << "with incorrect payload = " << actual_payload; - return false; - } - return true; -} - -// JitterWithReflectionStrategy child class which injects a specified stream -// of "random" numbers. -// -// Calculators are created through factory methods, making testing and injection -// tricky. This class utilizes a static variable, random_sequence, to pass -// the desired random sequence into the calculator. -class ReproducibleJitterWithReflectionStrategyForTesting - : public ReproducibleJitterWithReflectionStrategy { - public: - ReproducibleJitterWithReflectionStrategyForTesting( - PacketResamplerCalculator* calculator) - : ReproducibleJitterWithReflectionStrategy(calculator) {} - - // Statically accessed random sequence to use for jitter with reflection. - // - // An EXPECT will fail if sequence is less than the number requested during - // processing. - static std::vector random_sequence; - - protected: - virtual uint64 GetNextRandom(uint64 n) { - EXPECT_LT(sequence_index_, random_sequence.size()); - return random_sequence[sequence_index_++] % n; - } - - private: - int32 sequence_index_ = 0; -}; -std::vector - ReproducibleJitterWithReflectionStrategyForTesting::random_sequence; - -// PacketResamplerCalculator child class which injects a specified stream -// of "random" numbers. -// -// Calculators are created through factory methods, making testing and injection -// tricky. This class utilizes a static variable, random_sequence, to pass -// the desired random sequence into the calculator. -class ReproducibleResamplerCalculatorForTesting - : public PacketResamplerCalculator { - public: - static absl::Status GetContract(CalculatorContract* cc) { - return PacketResamplerCalculator::GetContract(cc); - } - - protected: - std::unique_ptr GetSamplingStrategy( - const mediapipe::PacketResamplerCalculatorOptions& Options) { - return absl::make_unique< - ReproducibleJitterWithReflectionStrategyForTesting>(this); - } -}; - -REGISTER_CALCULATOR(ReproducibleResamplerCalculatorForTesting); - -int SimpleRunner::static_count_ = 0; - -TEST(PacketResamplerCalculatorTest, NoPacketsInStream) { - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({}); - MP_ASSERT_OK(runner.Run()); - } -} - -TEST(PacketResamplerCalculatorTest, SinglePacketInStream) { - // Stream with 1 packet / 1 period. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0}, {0}); - } - - // Stream with 1 packet / 1 period (0 < packet timestamp < first limit). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({1000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({1000}, {1000}); - } - - // Stream with 1 packet / 1 period (packet timestamp > first limit). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({16668}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({16668}, {16668}); - } -} - -TEST(PacketResamplerCalculatorTest, TwoPacketsInStream) { - // Stream with 2 packets / 1 period. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 16666}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0}, {0}); - } - - // Stream with 2 packets / 2 periods (left extreme for second period). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 16667}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 16667}, {0, 33333}); - } - - // Stream with 2 packets / 2 periods (right extreme for second period). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 49999}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 49999}, {0, 33333}); - } - - // Stream with 2 packets / 3 periods (filling 1 in the middle). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 50000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 0, 50000}, {0, 33333, 66667}); - } - - // Stream with 2 packets / 4 periods (filling 2 in the middle). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({2000, 118666}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({2000, 2000, 2000, 118666}, - {2000, 35333, 68667, 102000}); - } -} - -TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepoints) { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 33333, 66667, 100000, 133333, 166667, 200000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps( - {0, 33333, 66667, 100000, 133333, 166667, 200000}, - {0, 33333, 66667, 100000, 133333, 166667, 200000}); -} - -// When there are several candidates for a period, the one closer to the center -// should be sent to the output. -TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriods) { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 16666, 16667, 20000, 33300, 49999, 50000, 66600}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 33300, 66600}, {0, 33333, 66667}); -} - -// When a period must be filled, we use the latest packet received (not -// necessarily the same as the one stored for the best in the previous period). -TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacket) { - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 5000, 16666, 83334}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 16666, 16666, 83334}, - {0, 33333, 66667, 100000}); - } - - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 16666, 16667, 25000, 33000, 35000, 135000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 33000, 35000, 35000, 135000}, - {0, 33333, 66667, 100000, 133333}); - } - - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({0, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 32000, 49999, 49999, 49999, 150000}, - {0, 33333, 66667, 100000, 133333, 166667}); - } -} - -TEST(PacketResamplerCalculatorTest, SuperHighFrameRate) { - // frame rate == 500000 (a packet will have to be sent every 2 ticks). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:500000}"); - runner.SetInput({0, 10, 13}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 0, 0, 0, 0, 10, 10, 13}, - {0, 2, 4, 6, 8, 10, 12, 14}); - } - - // frame rate == 1000000 (a packet will have to be sent in each tick). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:1000000}"); - runner.SetInput({0, 10, 13}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps( - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 13}, - {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}); - } -} - -TEST(PacketResamplerCalculatorTest, NegativeTimestampTest) { - // Stream with negative timestamps / 1 period. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-200, -20, 16466}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-200}, {-200}); - } - - // Stream with negative timestamps / 2 periods. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-200, -20, 16467}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-200, 16467}, {-200, 33133}); - } - - // Stream with negative timestamps and filling an empty period. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-500, 66667}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-500, -500, 66667}, {-500, 32833, 66167}); - } - - // Stream with negative timestamps and initial packet < -period. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-50000, -33334, 33334}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-50000, -33334, -33334, 33334}, - {-50000, -16667, 16667, 50000}); - } -} - -TEST(PacketResamplerCalculatorTest, ExactFramesPerSecond) { - // Using frame_rate=50, that makes a period of 20000 microsends (exact). - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:50}"); - runner.SetInput({0, 9999, 29999}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 29999}, {0, 20000}); - } - - // Test filling empty periods. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:50}"); - runner.SetInput({0, 10000, 50000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 10000, 10000, 50000}, - {0, 20000, 40000, 60000}); - } -} - -TEST(PacketResamplerCalculatorTest, FrameRateTest) { - // Test changing Frame Rate to the same initial value. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}"); - runner.SetInput({0, 10000, 30000, 50000, 60000}); - runner.SetVideoHeader(50.0); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 10000, 30000, 60000}, - {0, 20000, 40000, 60000}); - runner.CheckVideoHeader(50.0); - } - - // Test changing Frame Rate to new value. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:50, output_header:UPDATE_VIDEO_HEADER}"); - runner.SetInput({0, 5000, 10010, 15001, 19990}); - runner.SetVideoHeader(200.0); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 19990}, {0, 20000}); - runner.CheckVideoHeader(50.0); - } - - // Test that the frame rate is not changing if update_video_header = false. - { - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:50, output_header:PASS_HEADER}"); - runner.SetInput({0, 5000, 10010, 15001, 19990}); - runner.SetVideoHeader(200.0); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({0, 19990}, {0, 20000}); - runner.CheckVideoHeader(200.0); - } -} - -TEST(PacketResamplerCalculatorTest, SetVideoHeader) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "PacketResamplerCalculator" - input_stream: "DATA:in_data" - input_stream: "VIDEO_HEADER:in_video_header" - output_stream: "DATA:out_data" - output_stream: "VIDEO_HEADER:out_video_header" - options { - [mediapipe.PacketResamplerCalculatorOptions.ext] { frame_rate: 50.0 } - } - )pb")); - - for (const int64 ts : {0, 5000, 10010, 15001, 19990}) { - runner.MutableInputs()->Tag("DATA").packets.push_back( - Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts))); - } - VideoHeader video_header_in; - video_header_in.width = 10; - video_header_in.height = 100; - video_header_in.frame_rate = 1.0; - video_header_in.duration = 1.0; - video_header_in.format = ImageFormat::SRGB; - runner.MutableInputs() - ->Tag("VIDEO_HEADER") - .packets.push_back( - Adopt(new VideoHeader(video_header_in)).At(Timestamp::PreStream())); - MP_ASSERT_OK(runner.Run()); - - ASSERT_EQ(1, runner.Outputs().Tag("VIDEO_HEADER").packets.size()); - EXPECT_EQ(Timestamp::PreStream(), - runner.Outputs().Tag("VIDEO_HEADER").packets[0].Timestamp()); - const VideoHeader& video_header_out = - runner.Outputs().Tag("VIDEO_HEADER").packets[0].Get(); - EXPECT_EQ(video_header_in.width, video_header_out.width); - EXPECT_EQ(video_header_in.height, video_header_out.height); - EXPECT_DOUBLE_EQ(50.0, video_header_out.frame_rate); - EXPECT_FLOAT_EQ(video_header_in.duration, video_header_out.duration); - EXPECT_EQ(video_header_in.format, video_header_out.format); -} - -TEST(PacketResamplerCalculatorTest, FlushLastPacketWithoutRound) { - SimpleRunner runner(R"( - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 1 - })"); - runner.SetInput({0, 333333, 666667, 1000000, 1333333}); - MP_ASSERT_OK(runner.Run()); - // 1333333 is not emitted as 2000000, because it does not round to 2000000. - runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000}); -} - -TEST(PacketResamplerCalculatorTest, FlushLastPacketWithRound) { - SimpleRunner runner(R"( - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 1 - })"); - runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667}); - MP_ASSERT_OK(runner.Run()); - // 1666667 is emitted as 2000000, because it rounds to 2000000. - runner.CheckOutputTimestamps({0, 1000000, 1666667}, {0, 1000000, 2000000}); -} - -TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithoutRound) { - SimpleRunner runner(R"( - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 1 - flush_last_packet: false - })"); - runner.SetInput({0, 333333, 666667, 1000000, 1333333}); - MP_ASSERT_OK(runner.Run()); - // 1333333 is not emitted no matter what; see FlushLastPacketWithoutRound. - runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000}); -} - -TEST(PacketResamplerCalculatorTest, DoNotFlushLastPacketWithRound) { - SimpleRunner runner(R"( - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 1 - flush_last_packet: false - })"); - runner.SetInput({0, 333333, 666667, 1000000, 1333333, 1666667}); - MP_ASSERT_OK(runner.Run()); - // 1666667 is not emitted due to flush_last_packet: false. - runner.CheckOutputTimestamps({0, 1000000}, {0, 1000000}); -} - -// When base_timestamp is specified, output timestamps are aligned with it. -TEST(PacketResamplerCalculatorTest, InputAtExactFrequencyMiddlepointsAligned) { - { - // Without base_timestamp, outputs are aligned with the first input - // timestamp, (33333 - 222). - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({33111, 66667, 100000, 133333, 166667, 200000}, - {33111, 66444, 99778, 133111, 166444, 199778}); - } - { - // With base_timestamp, outputs are aligned with base_timestamp, 0. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0}"); - runner.SetInput({33111, 66667, 100000, 133333, 166667, 200000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps( - {33111, 66667, 100000, 133333, 166667, 200000}, - {33333, 66666, 100000, 133333, 166666, 200000}); - } -} - -// When base_timestamp is specified, output timestamps are aligned with it. -TEST(PacketResamplerCalculatorTest, MultiplePacketsForPeriodsAligned) { - { - // Without base_timestamp, outputs are aligned with the first input, -222. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-222, 33300, 66600}, {-222, 33111, 66445}); - } - { - // With base_timestamp, outputs are aligned with base_timestamp, 900011. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:900011}"); - runner.SetInput({-222, 16666, 16667, 20000, 33300, 49999, 50000, 66600}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-222, 33300, 66600}, {11, 33344, 66678}); - } - { - // With base_timestamp, outputs still approximate input timestamps, - // while aligned to base_timestamp, 11. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:11}"); - runner.SetInput( - {899888, 916666, 916667, 920000, 933300, 949999, 950000, 966600}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({899888, 933300, 966600}, - {900011, 933344, 966678}); - } -} - -// When a period must be filled, we use the latest packet received. -// When base_timestamp is specified, output timestamps are aligned with it. -TEST(PacketResamplerCalculatorTest, FillPeriodsWithLatestPacketAligned) { - { - // Without base_timestamp, outputs are aligned with the first input, -222. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30}"); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000}, - {-222, 33111, 66445, 99778, 133111, 166445}); - } - { - // With base_timestamp, outputs are aligned with base_timestamp, 0. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0}"); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000}, - {0, 33333, 66667, 100000, 133333, 166667}); - } -} - -// When base_timestamp is specified, output timestamps are aligned with it. -// The first packet is included, because we assume that the input includes the -// whole first sampling interval. -TEST(PacketResamplerCalculatorTest, FirstInputAfterMiddlepointAligned) { - { - // Packet 100020 is omitted from the output sequence because - // packet 99990 is closer to the period midpoint. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0}"); - runner.SetInput({66667, 100020, 133333, 166667}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({66667, 100020, 133333, 166667}, - {66667, 100000, 133334, 166667}); - } - { - // If we seek to packet 100020, packet 100020 is included in - // the output sequence, because we assume that the input includes the - // whole first sampling interval. - // - // We assume that the input includes whole sampling intervals - // in order to produce "reproducible timestamps", which are timestamps - // from the series of timestamps starting at 0. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0}"); - runner.SetInput({100020, 133333, 166667}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({100020, 133333, 166667}, - {100000, 133333, 166667}); - } -} - -TEST(PacketResamplerCalculatorTest, OutputTimestampRangeAligned) { - { - // With base_timestamp, outputs are aligned with base_timestamp, 0. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0}"); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({-222, 32000, 49999, 49999, 49999, 150000}, - {0, 33333, 66667, 100000, 133333, 166667}); - } - { - // With start_time, end_time, outputs are filtered. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0 " - "start_time:40000 " - "end_time:160000}"); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({49999, 49999, 49999}, - {66667, 100000, 133333}); - } - { - // With start_time, end_time, round_limits, outputs are filtered, - // rounding to the nearest limit. - SimpleRunner runner( - "[mediapipe.PacketResamplerCalculatorOptions.ext]: " - "{frame_rate:30 " - "base_timestamp:0 " - "start_time:40000 " - "end_time:160000 " - "round_limits:true}"); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - runner.CheckOutputTimestamps({32000, 49999, 49999, 49999, 150000}, - {33333, 66667, 100000, 133333, 166667}); - } -} - -TEST(PacketResamplerCalculatorTest, OptionsSidePacket) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "PacketResamplerCalculator" - input_side_packet: "OPTIONS:options" - input_stream: "input" - output_stream: "output" - options { - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 60 - base_timestamp: 0 - } - })pb"); - - { - SimpleRunner runner(node_config); - auto options = - new CalculatorOptions(ParseTextProtoOrDie( - R"pb( - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 30 - })pb")); - runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options); - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - EXPECT_EQ(6, runner.Outputs().Index(0).packets.size()); - } - { - SimpleRunner runner(node_config); - - auto options = - new CalculatorOptions(ParseTextProtoOrDie(R"pb( - merge_fields: false - [mediapipe.PacketResamplerCalculatorOptions.ext] { - frame_rate: 30 - base_timestamp: 0 - })pb")); - runner.MutableSidePackets()->Tag("OPTIONS") = Adopt(options); - - runner.SetInput({-222, 15000, 32000, 49999, 150000}); - MP_ASSERT_OK(runner.Run()); - EXPECT_EQ(6, runner.Outputs().Index(0).packets.size()); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_thinner_calculator.cc b/mediapipe/calculators/core/packet_thinner_calculator.cc deleted file mode 100644 index d3d391b61..000000000 --- a/mediapipe/calculators/core/packet_thinner_calculator.cc +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Declaration of PacketThinnerCalculator. - -#include // for ceil -#include - -#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/options_util.h" - -namespace mediapipe { - -namespace { -const double kTimebaseUs = 1000000; // Microseconds. -const char* const kOptionsTag = "OPTIONS"; -const char* const kPeriodTag = "PERIOD"; -} // namespace - -// This calculator is used to thin an input stream of Packets. -// An example application would be to sample decoded frames of video -// at a coarser temporal resolution. Unless otherwise stated, all -// timestamps are in units of microseconds. -// -// Thinning can be accomplished in one of two ways: -// 1) asynchronous thinning (known below as async): -// Algorithm does not rely on a master clock and is parameterized only -// by a single option -- the period. Once a packet is emitted, the -// thinner will discard subsequent packets for the duration of the period -// [Analogous to a refractory period during which packet emission is -// suppressed.] -// Packets arriving before start_time are discarded, as are packets -// arriving at or after end_time. -// 2) synchronous thinning (known below as sync): -// There are two variants of this algorithm, both parameterized by a -// start_time and a period. As in (1), packets arriving before start_time -// or at/after end_time are discarded. Otherwise, at most one packet is -// emitted during a period, centered at timestamps generated by the -// expression: -// start_time + i * period [where i is a non-negative integer] -// During each period, the packet closest to the generated timestamp is -// emitted (latest in the case of ties). In the first variant -// (sync_output_timestamps = true), the emitted packet is output at the -// generated timestamp. In the second variant, the packet is output at -// its original timestamp. Both variants emit exactly the same packets, -// but at different timestamps. -// -// Thinning period can be provided in the calculator options or via a -// side packet with the tag "PERIOD". -// -// Calculator options provided optionally with the "OPTIONS" input -// sidepacket tag will be merged with this calculator's node options, i.e., -// singular fields of the side packet will overwrite the options defined in the -// node, and repeated fields will concatenate. -// -// Example config: -// node { -// calculator: "PacketThinnerCalculator" -// input_side_packet: "OPTIONS:calculator_options" -// input_stream: "signal" -// output_stream: "output" -// options { -// [mediapipe.PacketThinnerCalculatorOptions.ext] { -// thinner_type: SYNC -// period: 10 -// sync_output_timestamps: true -// update_frame_rate: false -// } -// } -// } -class PacketThinnerCalculator : public CalculatorBase { - public: - PacketThinnerCalculator() {} - ~PacketThinnerCalculator() override {} - - static absl::Status GetContract(CalculatorContract* cc) { - if (cc->InputSidePackets().HasTag(kOptionsTag)) { - cc->InputSidePackets().Tag(kOptionsTag).Set(); - } - cc->Inputs().Index(0).SetAny(); - cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0)); - if (cc->InputSidePackets().HasTag(kPeriodTag)) { - cc->InputSidePackets().Tag(kPeriodTag).Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override { - if (cc->InputTimestamp() < start_time_) { - return absl::OkStatus(); // Drop packets before start_time_. - } else if (cc->InputTimestamp() >= end_time_) { - if (!cc->Outputs().Index(0).IsClosed()) { - cc->Outputs() - .Index(0) - .Close(); // No more Packets will be output after end_time_. - } - return absl::OkStatus(); - } else { - return thinner_type_ == PacketThinnerCalculatorOptions::ASYNC - ? AsyncThinnerProcess(cc) - : SyncThinnerProcess(cc); - } - } - - private: - // Implementation of ASYNC and SYNC versions of thinner algorithm. - absl::Status AsyncThinnerProcess(CalculatorContext* cc); - absl::Status SyncThinnerProcess(CalculatorContext* cc); - - // Cached option. - PacketThinnerCalculatorOptions::ThinnerType thinner_type_; - - // Given a Timestamp, finds the closest sync Timestamp - // based on start_time_ and period_. This can be earlier or - // later than given Timestamp, but is guaranteed to be within - // half a period_. - Timestamp NearestSyncTimestamp(Timestamp now) const; - - // Cached option used by both async and sync thinners. - TimestampDiff period_; // Interval during which only one packet is emitted. - Timestamp start_time_; // Cached option - default Timestamp::Min() - Timestamp end_time_; // Cached option - default Timestamp::Max() - - // Only used by async thinner: - Timestamp next_valid_timestamp_; // Suppress packets until this timestamp. - - // Only used by sync thinner: - Packet saved_packet_; // Best packet not yet emitted. - bool sync_output_timestamps_; // Cached option. -}; -REGISTER_CALCULATOR(PacketThinnerCalculator); - -namespace { -TimestampDiff abs(TimestampDiff t) { return t < 0 ? -t : t; } -} // namespace - -absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) { - PacketThinnerCalculatorOptions options = mediapipe::tool::RetrieveOptions( - cc->Options(), cc->InputSidePackets(), - kOptionsTag); - - thinner_type_ = options.thinner_type(); - // This check enables us to assume only two thinner types exist in Process() - CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC || - thinner_type_ == PacketThinnerCalculatorOptions::SYNC) - << "Unsupported thinner type."; - - if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) { - // ASYNC thinner outputs packets with the same timestamp as their input so - // its safe to SetOffset(0). SYNC thinner manipulates timestamps of its - // output so we don't do this for that case. - cc->SetOffset(0); - } - - if (cc->InputSidePackets().HasTag(kPeriodTag)) { - period_ = - TimestampDiff(cc->InputSidePackets().Tag(kPeriodTag).Get()); - } else { - period_ = TimestampDiff(options.period()); - } - CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive."; - - if (options.has_start_time()) { - start_time_ = Timestamp(options.start_time()); - } else if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) { - start_time_ = Timestamp::Min(); - } else { - start_time_ = Timestamp(0); - } - - end_time_ = - options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max(); - CHECK_LT(start_time_, end_time_) - << "Invalid PacketThinner: start_time must be earlier than end_time"; - - sync_output_timestamps_ = options.sync_output_timestamps(); - - next_valid_timestamp_ = start_time_; - // Drop packets until this time. - cc->Outputs().Index(0).SetNextTimestampBound(start_time_); - - if (!cc->Inputs().Index(0).Header().IsEmpty()) { - if (options.update_frame_rate()) { - const VideoHeader& video_header = - cc->Inputs().Index(0).Header().Get(); - double new_frame_rate; - if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) { - new_frame_rate = - video_header.frame_rate / - ceil(video_header.frame_rate * options.period() / kTimebaseUs); - } else { - const double sampling_rate = kTimebaseUs / options.period(); - new_frame_rate = video_header.frame_rate < sampling_rate - ? video_header.frame_rate - : sampling_rate; - } - std::unique_ptr header(new VideoHeader); - header->format = video_header.format; - header->width = video_header.width; - header->height = video_header.height; - header->frame_rate = new_frame_rate; - cc->Outputs().Index(0).SetHeader(Adopt(header.release())); - } else { - cc->Outputs().Index(0).SetHeader(cc->Inputs().Index(0).Header()); - } - } - - return absl::OkStatus(); -} - -absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) { - // Emit any saved packets before quitting. - if (!saved_packet_.IsEmpty()) { - // Only sync thinner should have saved packets. - CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_); - if (sync_output_timestamps_) { - cc->Outputs().Index(0).AddPacket( - saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp()))); - } else { - cc->Outputs().Index(0).AddPacket(saved_packet_); - } - } - return absl::OkStatus(); -} - -absl::Status PacketThinnerCalculator::AsyncThinnerProcess( - CalculatorContext* cc) { - if (cc->InputTimestamp() >= next_valid_timestamp_) { - cc->Outputs().Index(0).AddPacket( - cc->Inputs().Index(0).Value()); // Emit current packet. - next_valid_timestamp_ = cc->InputTimestamp() + period_; - // Guaranteed not to emit packets seen during refractory period. - cc->Outputs().Index(0).SetNextTimestampBound(next_valid_timestamp_); - } - return absl::OkStatus(); -} - -absl::Status PacketThinnerCalculator::SyncThinnerProcess( - CalculatorContext* cc) { - if (saved_packet_.IsEmpty()) { - // If no packet has been saved, store the current packet. - saved_packet_ = cc->Inputs().Index(0).Value(); - cc->Outputs().Index(0).SetNextTimestampBound( - sync_output_timestamps_ ? NearestSyncTimestamp(cc->InputTimestamp()) - : cc->InputTimestamp()); - } else { - // Saved packet exists -- update or emit. - const Timestamp saved = saved_packet_.Timestamp(); - const Timestamp saved_sync = NearestSyncTimestamp(saved); - const Timestamp now = cc->InputTimestamp(); - const Timestamp now_sync = NearestSyncTimestamp(now); - CHECK_LE(saved_sync, now_sync); - if (saved_sync == now_sync) { - // Saved Packet is in same interval as current packet. - // Replace saved packet with current if it is at least as - // central as the saved packet wrt temporal interval. - // [We break ties in favor of fresher packets] - if (abs(now - now_sync) <= abs(saved - saved_sync)) { - saved_packet_ = cc->Inputs().Index(0).Value(); - } - } else { - // Saved packet is the best packet from earlier interval: emit! - if (sync_output_timestamps_) { - cc->Outputs().Index(0).AddPacket(saved_packet_.At(saved_sync)); - cc->Outputs().Index(0).SetNextTimestampBound(now_sync); - } else { - cc->Outputs().Index(0).AddPacket(saved_packet_); - cc->Outputs().Index(0).SetNextTimestampBound(now); - } - // Current packet is the first one we've seen from new interval -- save! - saved_packet_ = cc->Inputs().Index(0).Value(); - } - } - return absl::OkStatus(); -} - -Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const { - CHECK_NE(start_time_, Timestamp::Unset()) - << "Method only valid for sync thinner calculator."; - - // Computation is done using int64 arithmetic. No easy way to avoid - // since Timestamps don't support div and multiply. - const int64 now64 = now.Value(); - const int64 start64 = start_time_.Value(); - const int64 period64 = period_.Value(); - CHECK_LE(0, period64); - - // Round now64 to its closest interval (units of period64). - int64 sync64 = - (now64 - start64 + period64 / 2) / period64 * period64 + start64; - CHECK_LE(abs(now64 - sync64), period64 / 2) - << "start64: " << start64 << "; now64: " << now64 - << "; sync64: " << sync64; - - return Timestamp(sync64); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/packet_thinner_calculator.proto b/mediapipe/calculators/core/packet_thinner_calculator.proto deleted file mode 100644 index 6c69f3afd..000000000 --- a/mediapipe/calculators/core/packet_thinner_calculator.proto +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message PacketThinnerCalculatorOptions { - extend CalculatorOptions { - optional PacketThinnerCalculatorOptions ext = 288533508; - } - - enum ThinnerType { - ASYNC = 1; // Asynchronous thinner, described below [default]. - SYNC = 2; // Synchronous thinner, also described below. - } - optional ThinnerType thinner_type = 1 [default = ASYNC]; - - // The period (in microsecond) specifies the temporal interval during which - // only a single packet is emitted in the output stream. Has subtly different - // semantics depending on the thinner type, as follows. - // - // Async thinner: this option is a refractory period -- once a packet is - // emitted, we guarantee that no packets will be emitted for period ticks. - // - // Sync thinner: the period specifies a temporal interval during which - // only one packet is emitted. The emitted packet is guaranteed to be - // the one closest to the center of the temporal interval (no guarantee on - // how ties are broken). More specifically, - // intervals are centered at start_time + i * period - // (for non-negative integers i). - // Thus, each interval extends period/2 ticks before and after its center. - // Additionally, in the sync thinner any packets earlier than start_time - // are discarded and the thinner calls Close() once timestamp equals or - // exceeds end_time. - optional int64 period = 2 [default = 1]; - - // Packets before start_time and at/after end_time are discarded. - // Additionally, for a sync thinner, start time specifies the center of - // time invervals as described above and therefore should be set explicitly. - optional int64 start_time = 3; // If not specified, set to 0 for SYNC type, - // and set to Timestamp::Min() for ASYNC type. - optional int64 end_time = 4; // Set to Timestamp::Max() if not specified. - - // Whether the timestamps of packets emitted by sync thinner should - // correspond to the center of their corresponding temporal interval. - // If false, packets emitted using original timestamp (as in async thinner). - optional bool sync_output_timestamps = 5 [default = true]; - - // If true, update the frame rate in the header, if it's available, to an - // estimated frame rate due to the sampling. - optional bool update_frame_rate = 6 [default = false]; -} diff --git a/mediapipe/calculators/core/packet_thinner_calculator_test.cc b/mediapipe/calculators/core/packet_thinner_calculator_test.cc deleted file mode 100644 index 86fcc00f9..000000000 --- a/mediapipe/calculators/core/packet_thinner_calculator_test.cc +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -// A simple version of CalculatorRunner with built-in convenience methods for -// setting inputs from a vector and checking outputs against a vector of -// expected outputs. -class SimpleRunner : public CalculatorRunner { - public: - explicit SimpleRunner(const CalculatorOptions& options) - : CalculatorRunner("PacketThinnerCalculator", options) { - SetNumInputs(1); - SetNumOutputs(1); - SetNumInputSidePackets(0); - } - - explicit SimpleRunner(const CalculatorGraphConfig::Node& node) - : CalculatorRunner(node) {} - - void SetInput(const std::vector& timestamp_list) { - MutableInputs()->Index(0).packets.clear(); - for (const int ts : timestamp_list) { - MutableInputs()->Index(0).packets.push_back( - MakePacket(absl::StrCat("Frame #", ts)) - .At(Timestamp(ts))); - } - } - - void SetFrameRate(const double frame_rate) { - auto video_header = absl::make_unique(); - video_header->frame_rate = frame_rate; - MutableInputs()->Index(0).header = Adopt(video_header.release()); - } - - std::vector GetOutputTimestamps() const { - std::vector timestamps; - for (const Packet& packet : Outputs().Index(0).packets) { - timestamps.emplace_back(packet.Timestamp().Value()); - } - return timestamps; - } - - double GetFrameRate() const { - CHECK(!Outputs().Index(0).header.IsEmpty()); - return Outputs().Index(0).header.Get().frame_rate; - } -}; - -// Check that thinner respects start_time and end_time options. -// We only test with one thinner because the logic for start & end time -// handling is shared across both types of thinner in Process(). -TEST(PacketThinnerCalculatorTest, StartAndEndTimeTest) { - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_period(5); - extension->set_start_time(4); - extension->set_end_time(12); - SimpleRunner runner(options); - runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {5, 11}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, AsyncUniformStreamThinningTest) { - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_period(5); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 8, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, ASyncUniformStreamThinningTestBySidePacket) { - // Note: sync runner but outputting *original* timestamps. - CalculatorGraphConfig::Node node; - node.set_calculator("PacketThinnerCalculator"); - node.add_input_side_packet("PERIOD:period"); - node.add_input_stream("input_stream"); - node.add_output_stream("output_stream"); - auto* extension = node.mutable_options()->MutableExtension( - PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_start_time(0); - extension->set_sync_output_timestamps(false); - - SimpleRunner runner(node); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - runner.MutableSidePackets()->Tag("PERIOD") = MakePacket(5); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 8, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest1) { - // Note: sync runner but outputting *original* timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(false); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 6, 10, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTestBySidePacket1) { - // Note: sync runner but outputting *original* timestamps. - CalculatorGraphConfig::Node node; - node.set_calculator("PacketThinnerCalculator"); - node.add_input_side_packet("PERIOD:period"); - node.add_input_stream("input_stream"); - node.add_output_stream("output_stream"); - auto* extension = node.mutable_options()->MutableExtension( - PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_sync_output_timestamps(false); - - SimpleRunner runner(node); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - runner.MutableSidePackets()->Tag("PERIOD") = MakePacket(5); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 6, 10, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, SyncUniformStreamThinningTest2) { - // Same test but now with synced timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(true); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {0, 5, 10, 15}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -// Test: Given a stream with timestamps corresponding to first ten prime numbers -// and period of 5, confirm whether timestamps of thinner stream matches -// expectations. -TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest1) { - // ASYNC thinner. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_period(5); - SimpleRunner runner(options); - runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 7, 13, 19, 29}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, PrimeStreamThinningTest2) { - // SYNC with original timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(false); - SimpleRunner runner(options); - runner.SetInput({2, 3, 5, 7, 11, 13, 17, 19, 23, 29}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 5, 11, 17, 19, 23, 29}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -// Confirm that Calculator correctly handles boundary cases. -TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest1) { - // Odd period, negative start_time - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(-10); - extension->set_period(5); - extension->set_sync_output_timestamps(true); - SimpleRunner runner(options); - // Two timestamps falling on either side of a period boundary. - runner.SetInput({2, 3}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {0, 5}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, BoundaryTimestampTest2) { - // Even period, negative start_time, negative packet timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(-144); - extension->set_period(6); - extension->set_sync_output_timestamps(true); - SimpleRunner runner(options); - // Two timestamps falling on either side of a period boundary. - runner.SetInput({-4, -3, 8, 9}); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {-6, 0, 6, 12}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); -} - -TEST(PacketThinnerCalculatorTest, FrameRateTest1) { - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_period(5); - extension->set_update_frame_rate(true); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - runner.SetFrameRate(1000000.0 / 2); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 8, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); - // The true sampling period is 6. - EXPECT_DOUBLE_EQ(1000000.0 / 6, runner.GetFrameRate()); -} - -TEST(PacketThinnerCalculatorTest, FrameRateTest2) { - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::ASYNC); - extension->set_period(5); - extension->set_update_frame_rate(true); - SimpleRunner runner(options); - runner.SetInput({8, 16, 24, 32, 40, 48, 56}); - runner.SetFrameRate(1000000.0 / 8); - MP_ASSERT_OK(runner.Run()); - const std::vector expected_timestamps = {8, 16, 24, 32, 40, 48, 56}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); - // The true sampling period is still 8. - EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate()); -} - -TEST(PacketThinnerCalculatorTest, FrameRateTest3) { - // Note: sync runner but outputting *original* timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(false); - extension->set_update_frame_rate(true); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - runner.SetFrameRate(1000000.0 / 2); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {2, 6, 10, 14}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); - // The true (long-run) sampling period is 5. - EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate()); -} - -TEST(PacketThinnerCalculatorTest, FrameRateTest4) { - // Same test but now with synced timestamps. - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(true); - extension->set_update_frame_rate(true); - SimpleRunner runner(options); - runner.SetInput({2, 4, 6, 8, 10, 12, 14}); - runner.SetFrameRate(1000000.0 / 2); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {0, 5, 10, 15}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); - // The true (long-run) sampling period is 5. - EXPECT_DOUBLE_EQ(1000000.0 / 5, runner.GetFrameRate()); -} - -TEST(PacketThinnerCalculatorTest, FrameRateTest5) { - CalculatorOptions options; - auto* extension = - options.MutableExtension(PacketThinnerCalculatorOptions::ext); - extension->set_thinner_type(PacketThinnerCalculatorOptions::SYNC); - extension->set_start_time(0); - extension->set_period(5); - extension->set_sync_output_timestamps(true); - extension->set_update_frame_rate(true); - SimpleRunner runner(options); - runner.SetInput({8, 16, 24, 32, 40, 48, 56}); - runner.SetFrameRate(1000000.0 / 8); - MP_ASSERT_OK(runner.Run()); - - const std::vector expected_timestamps = {10, 15, 25, 30, 40, 50, 55}; - EXPECT_EQ(expected_timestamps, runner.GetOutputTimestamps()); - // The true (long-run) sampling period is 8. - EXPECT_DOUBLE_EQ(1000000.0 / 8, runner.GetFrameRate()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/pass_through_calculator.cc b/mediapipe/calculators/core/pass_through_calculator.cc deleted file mode 100644 index 197e1331a..000000000 --- a/mediapipe/calculators/core/pass_through_calculator.cc +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" - -namespace mediapipe { - -// A Calculator that simply passes its input Packets and header through, -// unchanged. The inputs may be specified by tag or index. The outputs -// must match the inputs exactly. Any number of input side packets may -// also be specified. If output side packets are specified, they must -// match the input side packets exactly and the Calculator passes its -// input side packets through, unchanged. Otherwise, the input side -// packets will be ignored (allowing PassThroughCalculator to be used to -// test internal behavior). Any options may be specified and will be -// ignored. -class PassThroughCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - if (!cc->Inputs().TagMap()->SameAs(*cc->Outputs().TagMap())) { - return absl::InvalidArgumentError( - "Input and output streams to PassThroughCalculator must use " - "matching tags and indexes."); - } - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - cc->Inputs().Get(id).SetAny(); - cc->Outputs().Get(id).SetSameAs(&cc->Inputs().Get(id)); - } - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->InputSidePackets().Get(id).SetAny(); - } - if (cc->OutputSidePackets().NumEntries() != 0) { - if (!cc->InputSidePackets().TagMap()->SameAs( - *cc->OutputSidePackets().TagMap())) { - return absl::InvalidArgumentError( - "Input and output side packets to PassThroughCalculator must use " - "matching tags and indexes."); - } - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).SetSameAs( - &cc->InputSidePackets().Get(id)); - } - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).Header().IsEmpty()) { - cc->Outputs().Get(id).SetHeader(cc->Inputs().Get(id).Header()); - } - } - if (cc->OutputSidePackets().NumEntries() != 0) { - for (CollectionItemId id = cc->InputSidePackets().BeginId(); - id < cc->InputSidePackets().EndId(); ++id) { - cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); - } - } - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - cc->GetCounter("PassThrough")->Increment(); - if (cc->Inputs().NumEntries() == 0) { - return tool::StatusStop(); - } - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (!cc->Inputs().Get(id).IsEmpty()) { - VLOG(3) << "Passing " << cc->Inputs().Get(id).Name() << " to " - << cc->Outputs().Get(id).Name() << " at " - << cc->InputTimestamp().DebugString(); - cc->Outputs().Get(id).AddPacket(cc->Inputs().Get(id).Value()); - } - } - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(PassThroughCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/previous_loopback_calculator.cc b/mediapipe/calculators/core/previous_loopback_calculator.cc deleted file mode 100644 index d67e6c061..000000000 --- a/mediapipe/calculators/core/previous_loopback_calculator.cc +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { -namespace api2 { - -// PreviousLoopbackCalculator is useful when a graph needs to process an input -// together with some previous output. -// -// For the first packet that arrives on the MAIN input, the timestamp bound is -// advanced on the PREV_LOOP. Downstream calculators will see this as an empty -// packet. This way they are not kept waiting for the previous output, which -// for the first iteration does not exist. -// -// Thereafter, -// - Each non-empty MAIN packet results in: -// a) a PREV_LOOP packet with contents of the LOOP packet received at the -// timestamp of the previous non-empty MAIN packet -// b) or in a PREV_LOOP timestamp bound update if the LOOP packet was empty. -// - Each empty MAIN packet indicating timestamp bound update results in a -// PREV_LOOP timestamp bound update. -// -// Example config: -// node { -// calculator: "PreviousLoopbackCalculator" -// input_stream: "MAIN:input" -// input_stream: "LOOP:output" -// input_stream_info: { tag_index: 'LOOP' back_edge: true } -// output_stream: "PREV_LOOP:prev_output" -// } -// node { -// calculator: "FaceTracker" -// input_stream: "VIDEO:input" -// input_stream: "PREV_TRACK:prev_output" -// output_stream: "TRACK:output" -// } -class PreviousLoopbackCalculator : public Node { - public: - static constexpr Input kMain{"MAIN"}; - static constexpr Input kLoop{"LOOP"}; - static constexpr Output> kPrevLoop{"PREV_LOOP"}; - // TODO: an optional PREV_TIMESTAMP output could be added to - // carry the original timestamp of the packet on PREV_LOOP. - - MEDIAPIPE_NODE_CONTRACT(kMain, kLoop, kPrevLoop, - StreamHandler("ImmediateInputStreamHandler"), - TimestampChange::Arbitrary()); - - static absl::Status UpdateContract(CalculatorContract* cc) { - // Process() function is invoked in response to MAIN/LOOP stream timestamp - // bound updates. - cc->SetProcessTimestampBounds(true); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - kPrevLoop(cc).SetHeader(kLoop(cc).Header()); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - // Non-empty packets and empty packets indicating timestamp bound updates - // are guaranteed to have timestamps greater than timestamps of previous - // packets within the same stream. Calculator tracks and operates on such - // packets. - - const PacketBase& main_packet = kMain(cc).packet(); - if (prev_main_ts_ < main_packet.timestamp()) { - Timestamp loop_timestamp; - if (!main_packet.IsEmpty()) { - loop_timestamp = prev_non_empty_main_ts_; - prev_non_empty_main_ts_ = main_packet.timestamp(); - } else { - // Calculator advances PREV_LOOP timestamp bound in response to empty - // MAIN packet, hence not caring about corresponding loop packet. - loop_timestamp = Timestamp::Unset(); - } - main_packet_specs_.push_back({main_packet.timestamp(), loop_timestamp}); - prev_main_ts_ = main_packet.timestamp(); - } - - const PacketBase& loop_packet = kLoop(cc).packet(); - if (prev_loop_ts_ < loop_packet.timestamp()) { - loop_packets_.push_back(loop_packet); - prev_loop_ts_ = loop_packet.timestamp(); - } - - while (!main_packet_specs_.empty() && !loop_packets_.empty()) { - // The earliest MAIN packet. - MainPacketSpec main_spec = main_packet_specs_.front(); - // The earliest LOOP packet. - const PacketBase& loop_candidate = loop_packets_.front(); - // Match LOOP and MAIN packets. - if (main_spec.loop_timestamp < loop_candidate.timestamp()) { - // No LOOP packet can match the MAIN packet under review. - kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1); - main_packet_specs_.pop_front(); - } else if (main_spec.loop_timestamp > loop_candidate.timestamp()) { - // No MAIN packet can match the LOOP packet under review. - loop_packets_.pop_front(); - } else { - // Exact match found. - if (loop_candidate.IsEmpty()) { - // However, LOOP packet is empty. - kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1); - } else { - kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp)); - } - loop_packets_.pop_front(); - main_packet_specs_.pop_front(); - } - - // We can close PREV_LOOP output stream as soon as we processed last - // possible MAIN packet. That can happen in two cases: - // a) Non-empty MAIN packet has been received with Timestamp::Max() - // b) Empty MAIN packet has been received with Timestamp::Max() indicating - // MAIN is done. - if (main_spec.timestamp == Timestamp::Done().PreviousAllowedInStream()) { - kPrevLoop(cc).Close(); - } - } - - return absl::OkStatus(); - } - - private: - struct MainPacketSpec { - Timestamp timestamp; - // Expected timestamp of the packet from LOOP stream that corresponds to the - // packet from MAIN stream descirbed by this spec. - Timestamp loop_timestamp; - }; - - // Contains specs for MAIN packets which only can be: - // - non-empty packets - // - empty packets indicating timestamp bound updates - // - // Sorted according to packet timestamps. - std::deque main_packet_specs_; - Timestamp prev_main_ts_ = Timestamp::Unstarted(); - Timestamp prev_non_empty_main_ts_ = Timestamp::Unstarted(); - - // Contains LOOP packets which only can be: - // - the very first empty packet - // - non empty packets - // - empty packets indicating timestamp bound updates - // - // Sorted according to packet timestamps. - std::deque loop_packets_; - // Using "Timestamp::Unset" instead of "Timestamp::Unstarted" in order to - // allow addition of the very first empty packet (which doesn't indicate - // timestamp bound change necessarily). - Timestamp prev_loop_ts_ = Timestamp::Unset(); -}; -MEDIAPIPE_REGISTER_NODE(PreviousLoopbackCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/previous_loopback_calculator_test.cc b/mediapipe/calculators/core/previous_loopback_calculator_test.cc deleted file mode 100644 index c9d431d1c..000000000 --- a/mediapipe/calculators/core/previous_loopback_calculator_test.cc +++ /dev/null @@ -1,867 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "absl/time/clock.h" -#include "absl/time/time.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/sink.h" - -namespace mediapipe { - -using ::testing::ElementsAre; -using ::testing::Eq; -using ::testing::Pair; -using ::testing::Value; -namespace { - -// Returns the timestamp values for a vector of Packets. -// TODO: puth this kind of test util in a common place. -std::vector TimestampValues(const std::vector& packets) { - std::vector result; - for (const Packet& packet : packets) { - result.push_back(packet.Timestamp().Value()); - } - return result; -} - -MATCHER(EmptyPacket, negation ? "isn't empty" : "is empty") { - if (arg.IsEmpty()) { - return true; - } - return false; -} - -MATCHER_P(IntPacket, value, "") { - return Value(arg.template Get(), Eq(value)); -} - -MATCHER_P2(PairPacket, timestamp, pair, "") { - Timestamp actual_timestamp = arg.Timestamp(); - const auto& actual_pair = arg.template Get>(); - return Value(actual_timestamp, Eq(timestamp)) && Value(actual_pair, pair); -} - -TEST(PreviousLoopbackCalculator, CorrectTimestamps) { - std::vector in_prev; - CalculatorGraphConfig graph_config_ = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:in' - input_stream: 'LOOP:out' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:previous' - } - # This calculator synchronizes its inputs as normal, so it is used - # to check that both "in" and "previous" are ready. - node { - calculator: 'PassThroughCalculator' - input_stream: 'in' - input_stream: 'previous' - output_stream: 'out' - output_stream: 'previous2' - } - node { - calculator: 'MakePairCalculator' - input_stream: 'out' - input_stream: 'previous2' - output_stream: 'pair' - } - )pb"); - tool::AddVectorSink("pair", &graph_config_, &in_prev); - - CalculatorGraph graph_; - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - MP_ASSERT_OK(graph_.StartRun({})); - - auto send_packet = [&graph_](const std::string& input_name, int n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - - send_packet("in", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(in_prev), ElementsAre(1)); - EXPECT_THAT(in_prev.back(), - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket()))); - - send_packet("in", 2); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(in_prev), ElementsAre(1, 2)); - EXPECT_THAT(in_prev.back(), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1)))); - - send_packet("in", 5); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(in_prev), ElementsAre(1, 2, 5)); - EXPECT_THAT(in_prev.back(), - PairPacket(Timestamp(5), Pair(IntPacket(5), IntPacket(2)))); - - send_packet("in", 15); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(in_prev), ElementsAre(1, 2, 5, 15)); - EXPECT_THAT(in_prev.back(), - PairPacket(Timestamp(15), Pair(IntPacket(15), IntPacket(5)))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -// A Calculator that outputs a summary packet in CalculatorBase::Close(). -class PacketOnCloseCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - sum_ += cc->Inputs().Index(0).Value().Get(); - cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value()); - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) final { - cc->Outputs().Index(0).AddPacket( - MakePacket(sum_).At(Timestamp::Max())); - return absl::OkStatus(); - } - - private: - int sum_ = 0; -}; -REGISTER_CALCULATOR(PacketOnCloseCalculator); - -// Demonstrates that all ouput and input streams in PreviousLoopbackCalculator -// will close as expected when all graph input streams are closed. -TEST(PreviousLoopbackCalculator, ClosesCorrectly) { - std::vector outputs; - CalculatorGraphConfig graph_config_ = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:in' - input_stream: 'LOOP:out' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:previous' - } - # This calculator synchronizes its inputs as normal, so it is used - # to check that both "in" and "previous" are ready. - node { - calculator: 'PassThroughCalculator' - input_stream: 'in' - input_stream: 'previous' - output_stream: 'out' - output_stream: 'previous2' - } - node { - calculator: 'PacketOnCloseCalculator' - input_stream: 'out' - output_stream: 'close_out' - } - )pb"); - tool::AddVectorSink("close_out", &graph_config_, &outputs); - - CalculatorGraph graph_; - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - MP_ASSERT_OK(graph_.StartRun({})); - - auto send_packet = [&graph_](const std::string& input_name, int n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - - send_packet("in", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(outputs), ElementsAre(1)); - - send_packet("in", 2); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(outputs), ElementsAre(1, 2)); - - send_packet("in", 5); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(outputs), ElementsAre(1, 2, 5)); - - send_packet("in", 15); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(outputs), ElementsAre(1, 2, 5, 15)); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT(TimestampValues(outputs), - ElementsAre(1, 2, 5, 15, Timestamp::Max().Value())); - - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST(PreviousLoopbackCalculator, ProcessesMaxTimestamp) { - std::vector out_and_previous_packets; - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:in' - input_stream: 'LOOP:out' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:previous' - } - node { - calculator: 'PassThroughCalculator' - input_stream: 'in' - input_stream: 'previous' - output_stream: 'out' - output_stream: 'previous2' - } - node { - calculator: 'MakePairCalculator' - input_stream: 'out' - input_stream: 'previous' - output_stream: 'out_and_previous' - } - )pb"); - tool::AddVectorSink("out_and_previous", &graph_config, - &out_and_previous_packets); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config, {})); - MP_ASSERT_OK(graph.StartRun({})); - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in", MakePacket(1).At(Timestamp::Max()))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - - EXPECT_THAT(out_and_previous_packets, - ElementsAre(PairPacket(Timestamp::Max(), - Pair(IntPacket(1), EmptyPacket())))); - - MP_EXPECT_OK(graph.CloseAllInputStreams()); - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.WaitUntilDone()); -} - -TEST(PreviousLoopbackCalculator, ProcessesMaxTimestampNonEmptyPrevious) { - std::vector out_and_previous_packets; - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:in' - input_stream: 'LOOP:out' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:previous' - } - node { - calculator: 'PassThroughCalculator' - input_stream: 'in' - input_stream: 'previous' - output_stream: 'out' - output_stream: 'previous2' - } - node { - calculator: 'MakePairCalculator' - input_stream: 'out' - input_stream: 'previous' - output_stream: 'out_and_previous' - } - )pb"); - tool::AddVectorSink("out_and_previous", &graph_config, - &out_and_previous_packets); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config, {})); - MP_ASSERT_OK(graph.StartRun({})); - - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in", MakePacket(1).At(Timestamp::Min()))); - MP_EXPECT_OK(graph.AddPacketToInputStream( - "in", MakePacket(2).At(Timestamp::Max()))); - - MP_EXPECT_OK(graph.WaitUntilIdle()); - - EXPECT_THAT( - out_and_previous_packets, - ElementsAre( - PairPacket(Timestamp::Min(), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp::Max(), Pair(IntPacket(2), IntPacket(1))))); - - MP_EXPECT_OK(graph.CloseAllInputStreams()); - MP_EXPECT_OK(graph.WaitUntilIdle()); - MP_EXPECT_OK(graph.WaitUntilDone()); -} - -// Demonstrates that downstream calculators won't be blocked by -// always-empty-LOOP-stream. -TEST(PreviousLoopbackCalculator, EmptyLoopForever) { - std::vector outputs; - CalculatorGraphConfig graph_config_ = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:in' - input_stream: 'LOOP:previous' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:previous' - } - # This calculator synchronizes its inputs as normal, so it is used - # to check that both "in" and "previous" are ready. - node { - calculator: 'PassThroughCalculator' - input_stream: 'in' - input_stream: 'previous' - output_stream: 'out' - output_stream: 'previous2' - } - node { - calculator: 'PacketOnCloseCalculator' - input_stream: 'out' - output_stream: 'close_out' - } - )pb"); - tool::AddVectorSink("close_out", &graph_config_, &outputs); - - CalculatorGraph graph_; - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - MP_ASSERT_OK(graph_.StartRun({})); - - auto send_packet = [&graph_](const std::string& input_name, int n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - - for (int main_ts = 0; main_ts < 50; ++main_ts) { - send_packet("in", main_ts); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - std::vector ts_values = TimestampValues(outputs); - EXPECT_EQ(ts_values.size(), main_ts + 1); - for (int j = 0; j < main_ts + 1; ++j) { - EXPECT_EQ(ts_values[j], j); - } - } - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -class PreviousLoopbackCalculatorProcessingTimestampsTest - : public testing::Test { - protected: - void SetUp() override { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'input' - input_stream: 'force_main_empty' - input_stream: 'force_loop_empty' - # Used to indicate "main" timestamp bound updates. - node { - calculator: 'GateCalculator' - input_stream: 'input' - input_stream: 'DISALLOW:force_main_empty' - output_stream: 'main' - } - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:main' - input_stream: 'LOOP:loop' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:prev_loop' - } - node { - calculator: 'PassThroughCalculator' - input_stream: 'input' - input_stream: 'prev_loop' - output_stream: 'passed_through_input' - output_stream: 'passed_through_prev_loop' - } - # Used to indicate "loop" timestamp bound updates. - node { - calculator: 'GateCalculator' - input_stream: 'input' - input_stream: 'DISALLOW:force_loop_empty' - output_stream: 'loop' - } - node { - calculator: 'MakePairCalculator' - input_stream: 'passed_through_input' - input_stream: 'passed_through_prev_loop' - output_stream: 'passed_through_input_and_prev_loop' - } - )pb"); - tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config, - &output_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config, {})); - MP_ASSERT_OK(graph_.StartRun({})); - } - - void SendPackets(int timestamp, int input, bool force_main_empty, - bool force_loop_empty) { - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "input", MakePacket(input).At(Timestamp(timestamp)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "force_main_empty", - MakePacket(force_main_empty).At(Timestamp(timestamp)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "force_loop_empty", - MakePacket(force_loop_empty).At(Timestamp(timestamp)))); - } - - CalculatorGraph graph_; - std::vector output_packets_; -}; - -TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest, - MultiplePacketsEmptyMainNonEmptyLoop) { - SendPackets(/*timestamp=*/1, /*input=*/1, /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())))); - - SendPackets(/*timestamp=*/3, /*input=*/3, /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())))); - - SendPackets(/*timestamp=*/5, /*input=*/5, /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())))); - - SendPackets(/*timestamp=*/15, /*input=*/15, - /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())), - PairPacket(Timestamp(15), Pair(IntPacket(15), EmptyPacket())))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest, - MultiplePacketsNonEmptyMainEmptyLoop) { - SendPackets(/*timestamp=*/1, /*input=*/1, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())))); - - SendPackets(/*timestamp=*/3, /*input=*/3, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())))); - - SendPackets(/*timestamp=*/5, /*input=*/5, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())))); - - SendPackets(/*timestamp=*/15, /*input=*/15, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())), - PairPacket(Timestamp(15), Pair(IntPacket(15), EmptyPacket())))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest, - MultiplePacketsAlteringMainNonEmptyLoop) { - SendPackets(/*timestamp=*/1, /*input=*/1, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())))); - - SendPackets(/*timestamp=*/3, /*input=*/3, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), IntPacket(1))))); - - SendPackets(/*timestamp=*/5, /*input=*/5, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), IntPacket(1))), - PairPacket(Timestamp(5), Pair(IntPacket(5), IntPacket(3))))); - - SendPackets(/*timestamp=*/15, /*input=*/15, - /*force_main_empty=*/true, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), IntPacket(1))), - PairPacket(Timestamp(5), Pair(IntPacket(5), IntPacket(3))), - PairPacket(Timestamp(15), Pair(IntPacket(15), EmptyPacket())))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest, - MultiplePacketsNonEmptyMainAlteringLoop) { - SendPackets(/*timestamp=*/1, /*input=*/1, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))))); - - SendPackets(/*timestamp=*/3, /*input=*/3, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())))); - - SendPackets(/*timestamp=*/5, /*input=*/5, - /*force_main_empty=*/false, - /*force_loop_empty=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), IntPacket(3))))); - - SendPackets(/*timestamp=*/15, /*input=*/15, - /*force_main_empty=*/false, - /*force_loop_empty=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), IntPacket(3))), - PairPacket(Timestamp(15), Pair(IntPacket(15), EmptyPacket())))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST_F(PreviousLoopbackCalculatorProcessingTimestampsTest, - MultiplePacketsCheckIfLastCorrectAlteringMainAlteringLoop) { - int num_packets = 1000; - for (int i = 0; i < num_packets; ++i) { - bool force_main_empty = i % 3 == 0 ? true : false; - bool force_loop_empty = i % 2 == 0 ? true : false; - SendPackets(/*timestamp=*/i + 1, /*input=*/i + 1, force_main_empty, - force_loop_empty); - } - SendPackets(/*timestamp=*/num_packets + 1, - /*input=*/num_packets + 1, /*force_main_empty=*/false, - /*force_loop_empty=*/false); - SendPackets(/*timestamp=*/num_packets + 2, - /*input=*/num_packets + 2, /*force_main_empty=*/false, - /*force_loop_empty=*/false); - - MP_EXPECT_OK(graph_.WaitUntilIdle()); - ASSERT_FALSE(output_packets_.empty()); - EXPECT_THAT( - output_packets_.back(), - PairPacket(Timestamp(num_packets + 2), - Pair(IntPacket(num_packets + 2), IntPacket(num_packets + 1)))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -// Similar to GateCalculator, but it doesn't propagate timestamp bound updates. -class DroppingGateCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).SetAny(); - cc->Inputs().Tag("DISALLOW").Set(); - cc->Outputs().Index(0).SetSameAs(&cc->Inputs().Index(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - if (!cc->Inputs().Index(0).IsEmpty() && - !cc->Inputs().Tag("DISALLOW").Get()) { - cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value()); - } - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(DroppingGateCalculator); - -// Tests PreviousLoopbackCalculator in cases when there are no "LOOP" timestamp -// bound updates and non-empty packets for a while and the aforementioned start -// to arrive at some point. So, "PREV_LOOP" is delayed for a couple of inputs. -class PreviousLoopbackCalculatorDelayBehaviorTest : public testing::Test { - protected: - void SetUp() override { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(R"pb( - input_stream: 'input' - # Drops "loop" when set to "true", delaying output of prev_loop, hence - # delaying output of the graph. - input_stream: 'delay_next_output' - node { - calculator: 'PreviousLoopbackCalculator' - input_stream: 'MAIN:input' - input_stream: 'LOOP:loop' - input_stream_info: { tag_index: 'LOOP' back_edge: true } - output_stream: 'PREV_LOOP:prev_loop' - } - node { - calculator: 'PassThroughCalculator' - input_stream: 'input' - input_stream: 'prev_loop' - output_stream: 'passed_through_input' - output_stream: 'passed_through_prev_loop' - } - node { - calculator: 'DroppingGateCalculator' - input_stream: 'input' - input_stream: 'DISALLOW:delay_next_output' - output_stream: 'loop' - } - node { - calculator: 'MakePairCalculator' - input_stream: 'passed_through_input' - input_stream: 'passed_through_prev_loop' - output_stream: 'passed_through_input_and_prev_loop' - } - )pb"); - tool::AddVectorSink("passed_through_input_and_prev_loop", &graph_config, - &output_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config, {})); - MP_ASSERT_OK(graph_.StartRun({})); - } - - void SendPackets(int timestamp, int input, bool delay_next_output) { - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "input", MakePacket(input).At(Timestamp(timestamp)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delay_next_output", - MakePacket(delay_next_output).At(Timestamp(timestamp)))); - } - - CalculatorGraph graph_; - std::vector output_packets_; -}; - -TEST_F(PreviousLoopbackCalculatorDelayBehaviorTest, MultipleDelayedOutputs) { - SendPackets(/*timestamp=*/1, /*input=*/1, /*delay_next_output=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, /*delay_next_output=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/3, /*input=*/3, /*delay_next_output=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/5, /*input=*/5, /*delay_next_output=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())))); - - SendPackets(/*timestamp=*/15, /*input=*/15, /*delay_next_output=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), EmptyPacket())), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())), - PairPacket(Timestamp(15), Pair(IntPacket(15), IntPacket(5))))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST_F(PreviousLoopbackCalculatorDelayBehaviorTest, - NonDelayedOutputFollowedByMultipleDelayedOutputs) { - SendPackets(/*timestamp=*/1, /*input=*/1, /*delay_next_output=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())))); - - SendPackets(/*timestamp=*/2, /*input=*/2, /*delay_next_output=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))))); - - SendPackets(/*timestamp=*/3, /*input=*/3, /*delay_next_output=*/true); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))))); - - SendPackets(/*timestamp=*/5, /*input=*/5, /*delay_next_output=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre(PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())))); - - SendPackets(/*timestamp=*/15, /*input=*/15, /*delay_next_output=*/false); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_THAT( - output_packets_, - ElementsAre( - PairPacket(Timestamp(1), Pair(IntPacket(1), EmptyPacket())), - PairPacket(Timestamp(2), Pair(IntPacket(2), IntPacket(1))), - PairPacket(Timestamp(3), Pair(IntPacket(3), EmptyPacket())), - PairPacket(Timestamp(5), Pair(IntPacket(5), EmptyPacket())), - PairPacket(Timestamp(15), Pair(IntPacket(15), IntPacket(5))))); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -} // anonymous namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/quantize_float_vector_calculator.cc b/mediapipe/calculators/core/quantize_float_vector_calculator.cc deleted file mode 100644 index e95509298..000000000 --- a/mediapipe/calculators/core/quantize_float_vector_calculator.cc +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "mediapipe/calculators/core/quantize_float_vector_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/status.h" - -// Quantizes a vector of floats to a std::string so that each float becomes a -// byte in the [0, 255] range. Any value above max_quantized_value or below -// min_quantized_value will be saturated to '/xFF' or '/0'. -// -// Example config: -// node { -// calculator: "QuantizeFloatVectorCalculator" -// input_stream: "FLOAT_VECTOR:float_vector" -// output_stream: "ENCODED:encoded" -// options { -// [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { -// max_quantized_value: 64 -// min_quantized_value: -64 -// } -// } -// } -namespace mediapipe { - -class QuantizeFloatVectorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag("FLOAT_VECTOR").Set>(); - cc->Outputs().Tag("ENCODED").Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - const auto options = - cc->Options<::mediapipe::QuantizeFloatVectorCalculatorOptions>(); - if (!options.has_max_quantized_value() || - !options.has_min_quantized_value()) { - return absl::InvalidArgumentError( - "Both max_quantized_value and min_quantized_value must be provided " - "in QuantizeFloatVectorCalculatorOptions."); - } - max_quantized_value_ = options.max_quantized_value(); - min_quantized_value_ = options.min_quantized_value(); - if (max_quantized_value_ < min_quantized_value_ + FLT_EPSILON) { - return absl::InvalidArgumentError( - "max_quantized_value must be greater than min_quantized_value."); - } - range_ = max_quantized_value_ - min_quantized_value_; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) final { - const std::vector& float_vector = - cc->Inputs().Tag("FLOAT_VECTOR").Value().Get>(); - int feature_size = float_vector.size(); - std::string encoded_features; - encoded_features.reserve(feature_size); - for (int i = 0; i < feature_size; i++) { - float old_value = float_vector[i]; - if (old_value < min_quantized_value_) { - old_value = min_quantized_value_; - } - if (old_value > max_quantized_value_) { - old_value = max_quantized_value_; - } - unsigned char encoded = static_cast( - (old_value - min_quantized_value_) * (255.0 / range_)); - encoded_features += encoded; - } - cc->Outputs().Tag("ENCODED").AddPacket( - MakePacket(encoded_features).At(cc->InputTimestamp())); - return absl::OkStatus(); - } - - private: - float max_quantized_value_; - float min_quantized_value_; - float range_; -}; - -REGISTER_CALCULATOR(QuantizeFloatVectorCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/quantize_float_vector_calculator.proto b/mediapipe/calculators/core/quantize_float_vector_calculator.proto deleted file mode 100644 index 0ccc3c0d9..000000000 --- a/mediapipe/calculators/core/quantize_float_vector_calculator.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message QuantizeFloatVectorCalculatorOptions { - extend CalculatorOptions { - optional QuantizeFloatVectorCalculatorOptions ext = 259848061; - } - - optional float max_quantized_value = 1; - optional float min_quantized_value = 2; -} diff --git a/mediapipe/calculators/core/quantize_float_vector_calculator_test.cc b/mediapipe/calculators/core/quantize_float_vector_calculator_test.cc deleted file mode 100644 index 8f23437b6..000000000 --- a/mediapipe/calculators/core/quantize_float_vector_calculator_test.cc +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - min_quantized_value: 1 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector empty_vector; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(empty_vector).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "Both max_quantized_value and min_quantized_value must be provided")); -} - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig2) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - max_quantized_value: -1 - min_quantized_value: 1 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector empty_vector; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(empty_vector).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "max_quantized_value must be greater than min_quantized_value")); -} - -TEST(QuantizeFloatVectorCalculatorTest, WrongConfig3) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - max_quantized_value: 1 - min_quantized_value: 1 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector empty_vector; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(empty_vector).At(Timestamp(0))); - auto status = runner.Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - testing::HasSubstr( - "max_quantized_value must be greater than min_quantized_value")); -} - -TEST(QuantizeFloatVectorCalculatorTest, TestEmptyVector) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - max_quantized_value: 1 - min_quantized_value: -1 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector empty_vector; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(empty_vector).At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const std::vector& outputs = runner.Outputs().Tag("ENCODED").packets; - EXPECT_EQ(1, outputs.size()); - EXPECT_TRUE(outputs[0].Get().empty()); - EXPECT_EQ(Timestamp(0), outputs[0].Timestamp()); -} - -TEST(QuantizeFloatVectorCalculatorTest, TestNonEmptyVector) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - max_quantized_value: 64 - min_quantized_value: -64 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector vector = {0.0f, -64.0f, 64.0f, -32.0f, 32.0f}; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(vector).At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const std::vector& outputs = runner.Outputs().Tag("ENCODED").packets; - EXPECT_EQ(1, outputs.size()); - const std::string& result = outputs[0].Get(); - ASSERT_FALSE(result.empty()); - EXPECT_EQ(5, result.size()); - // 127 - EXPECT_EQ('\x7F', result.c_str()[0]); - // 0 - EXPECT_EQ('\0', result.c_str()[1]); - // 255 - EXPECT_EQ('\xFF', result.c_str()[2]); - // 63 - EXPECT_EQ('\x3F', result.c_str()[3]); - // 191 - EXPECT_EQ('\xBF', result.c_str()[4]); - EXPECT_EQ(Timestamp(0), outputs[0].Timestamp()); -} - -TEST(QuantizeFloatVectorCalculatorTest, TestSaturation) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "QuantizeFloatVectorCalculator" - input_stream: "FLOAT_VECTOR:float_vector" - output_stream: "ENCODED:encoded" - options { - [mediapipe.QuantizeFloatVectorCalculatorOptions.ext]: { - max_quantized_value: 64 - min_quantized_value: -64 - } - } - )pb"); - CalculatorRunner runner(node_config); - std::vector vector = {-65.0f, 65.0f}; - runner.MutableInputs() - ->Tag("FLOAT_VECTOR") - .packets.push_back( - MakePacket>(vector).At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const std::vector& outputs = runner.Outputs().Tag("ENCODED").packets; - EXPECT_EQ(1, outputs.size()); - const std::string& result = outputs[0].Get(); - ASSERT_FALSE(result.empty()); - EXPECT_EQ(2, result.size()); - // 0 - EXPECT_EQ('\0', result.c_str()[0]); - // 255 - EXPECT_EQ('\xFF', result.c_str()[1]); - EXPECT_EQ(Timestamp(0), outputs[0].Timestamp()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc b/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc deleted file mode 100644 index 277f83fe2..000000000 --- a/mediapipe/calculators/core/real_time_flow_limiter_calculator.cc +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/header_util.h" - -namespace mediapipe { - -// RealTimeFlowLimiterCalculator is used to limit the number of pipelined -// processing operations in a section of the graph. -// -// Typical topology: -// -// in ->-[FLC]-[foo]-...-[bar]-+->- out -// ^_____________________| -// FINISHED -// -// By connecting the output of the graph section to this calculator's FINISHED -// input with a backwards edge, this allows FLC to keep track of how many -// timestamps are currently being processed. -// -// The limit defaults to 1, and can be overridden with the MAX_IN_FLIGHT side -// packet. -// -// As long as the number of timestamps being processed ("in flight") is below -// the limit, FLC allows input to pass through. When the limit is reached, -// FLC starts dropping input packets, keeping only the most recent. When the -// processing count decreases again, as signaled by the receipt of a packet on -// FINISHED, FLC allows packets to flow again, releasing the most recently -// queued packet, if any. -// -// If there are multiple input streams, packet dropping is synchronized. -// -// IMPORTANT: for each timestamp where FLC forwards a packet (or a set of -// packets, if using multiple data streams), a packet must eventually arrive on -// the FINISHED stream. Dropping packets in the section between FLC and -// FINISHED will make the in-flight count incorrect. -// -// TODO: Remove this comment when graph-level ISH has been removed. -// NOTE: this calculator should always use the ImmediateInputStreamHandler and -// uses it by default. However, if the graph specifies a graph-level -// InputStreamHandler, to override that setting, the InputStreamHandler must -// be explicitly specified as shown below. -// -// Example config: -// node { -// calculator: "RealTimeFlowLimiterCalculator" -// input_stream: "raw_frames" -// input_stream: "FINISHED:finished" -// input_stream_info: { -// tag_index: 'FINISHED' -// back_edge: true -// } -// input_stream_handler { -// input_stream_handler: 'ImmediateInputStreamHandler' -// } -// output_stream: "gated_frames" -// } -class RealTimeFlowLimiterCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - int num_data_streams = cc->Inputs().NumEntries(""); - RET_CHECK_GE(num_data_streams, 1); - RET_CHECK_EQ(cc->Outputs().NumEntries(""), num_data_streams) - << "Output streams must correspond input streams except for the " - "finish indicator input stream."; - for (int i = 0; i < num_data_streams; ++i) { - cc->Inputs().Get("", i).SetAny(); - cc->Outputs().Get("", i).SetSameAs(&(cc->Inputs().Get("", i))); - } - cc->Inputs().Get("FINISHED", 0).SetAny(); - if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) { - cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Set(); - } - if (cc->Outputs().HasTag("ALLOW")) { - cc->Outputs().Tag("ALLOW").Set(); - } - - cc->SetInputStreamHandler("ImmediateInputStreamHandler"); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) final { - finished_id_ = cc->Inputs().GetId("FINISHED", 0); - max_in_flight_ = 1; - if (cc->InputSidePackets().HasTag("MAX_IN_FLIGHT")) { - max_in_flight_ = cc->InputSidePackets().Tag("MAX_IN_FLIGHT").Get(); - } - RET_CHECK_GE(max_in_flight_, 1); - num_in_flight_ = 0; - - allowed_id_ = cc->Outputs().GetId("ALLOW", 0); - allow_ctr_ts_ = Timestamp(0); - - num_data_streams_ = cc->Inputs().NumEntries(""); - data_stream_bound_ts_.resize(num_data_streams_); - RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &(cc->Outputs()))); - return absl::OkStatus(); - } - - bool Allow() { return num_in_flight_ < max_in_flight_; } - - absl::Status Process(CalculatorContext* cc) final { - bool old_allow = Allow(); - Timestamp lowest_incomplete_ts = Timestamp::Done(); - - // Process FINISHED stream. - if (!cc->Inputs().Get(finished_id_).Value().IsEmpty()) { - RET_CHECK_GT(num_in_flight_, 0) - << "Received a FINISHED packet, but we had none in flight."; - --num_in_flight_; - } - - // Process data streams. - for (int i = 0; i < num_data_streams_; ++i) { - auto& stream = cc->Inputs().Get("", i); - auto& out = cc->Outputs().Get("", i); - Packet& packet = stream.Value(); - auto ts = packet.Timestamp(); - if (ts.IsRangeValue() && data_stream_bound_ts_[i] <= ts) { - data_stream_bound_ts_[i] = ts + 1; - // Note: it's ok to update the output bound here, before sending the - // packet, because updates are batched during the Process function. - out.SetNextTimestampBound(data_stream_bound_ts_[i]); - } - lowest_incomplete_ts = - std::min(lowest_incomplete_ts, data_stream_bound_ts_[i]); - - if (packet.IsEmpty()) { - // If the input stream is closed, close the corresponding output. - if (stream.IsDone() && !out.IsClosed()) { - out.Close(); - } - // TODO: if the packet is empty, the ts is unset, and we - // cannot read the timestamp bound, even though we'd like to propagate - // it. - } else if (mediapipe::ContainsKey(pending_ts_, ts)) { - // If we have already sent this timestamp (on another stream), send it - // on this stream too. - out.AddPacket(std::move(packet)); - } else if (Allow() && (ts > last_dropped_ts_)) { - // If the in-flight is under the limit, and if we have not already - // dropped this or a later timestamp on another stream, then send - // the packet and add an in-flight timestamp. - out.AddPacket(std::move(packet)); - pending_ts_.insert(ts); - ++num_in_flight_; - } else { - // Otherwise, we'll drop the packet. - last_dropped_ts_ = std::max(last_dropped_ts_, ts); - } - } - - // Remove old pending_ts_ entries. - auto it = std::lower_bound(pending_ts_.begin(), pending_ts_.end(), - lowest_incomplete_ts); - pending_ts_.erase(pending_ts_.begin(), it); - - // Update ALLOW signal. - if ((old_allow != Allow()) && allowed_id_.IsValid()) { - cc->Outputs() - .Get(allowed_id_) - .AddPacket(MakePacket(Allow()).At(++allow_ctr_ts_)); - } - return absl::OkStatus(); - } - - private: - std::set pending_ts_; - Timestamp last_dropped_ts_; - int num_data_streams_; - int num_in_flight_; - int max_in_flight_; - CollectionItemId finished_id_; - CollectionItemId allowed_id_; - Timestamp allow_ctr_ts_; - std::vector data_stream_bound_ts_; -}; -REGISTER_CALCULATOR(RealTimeFlowLimiterCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc b/mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc deleted file mode 100644 index fe4785860..000000000 --- a/mediapipe/calculators/core/real_time_flow_limiter_calculator_test.cc +++ /dev/null @@ -1,495 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/time/clock.h" -#include "absl/time/time.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/sink.h" - -namespace mediapipe { - -namespace { -// A simple Semaphore for synchronizing test threads. -class AtomicSemaphore { - public: - AtomicSemaphore(int64_t supply) : supply_(supply) {} - void Acquire(int64_t amount) { - while (supply_.fetch_sub(amount) - amount < 0) { - Release(amount); - } - } - void Release(int64_t amount) { supply_.fetch_add(amount); } - - private: - std::atomic supply_; -}; - -// Returns the timestamp values for a vector of Packets. -std::vector TimestampValues(const std::vector& packets) { - std::vector result; - for (const Packet& packet : packets) { - result.push_back(packet.Timestamp().Value()); - } - return result; -} - -// Returns the packet values for a vector of Packets. -template -std::vector PacketValues(const std::vector& packets) { - std::vector result; - for (const Packet& packet : packets) { - result.push_back(packet.Get()); - } - return result; -} - -constexpr int kNumImageFrames = 5; -constexpr int kNumFinished = 3; -CalculatorGraphConfig::Node GetDefaultNode() { - return ParseTextProtoOrDie(R"pb( - calculator: "RealTimeFlowLimiterCalculator" - input_stream: "raw_frames" - input_stream: "FINISHED:finished" - input_stream_info: { tag_index: "FINISHED" back_edge: true } - output_stream: "gated_frames" - )pb"); -} - -// Simple test to make sure that the RealTimeFlowLimiterCalculator outputs just -// one packet when MAX_IN_FLIGHT is 1. -TEST(RealTimeFlowLimiterCalculator, OneOutputTest) { - // Setup the calculator runner and add only ImageFrame packets. - CalculatorRunner runner(GetDefaultNode()); - for (int i = 0; i < kNumImageFrames; ++i) { - Timestamp timestamp = Timestamp(i * Timestamp::kTimestampUnitsPerSecond); - runner.MutableInputs()->Index(0).packets.push_back( - MakePacket().At(timestamp)); - } - - // Run the calculator. - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& frame_output_packets = - runner.Outputs().Index(0).packets; - - EXPECT_EQ(frame_output_packets.size(), 1); -} - -// Simple test to make sure that the RealTimeFlowLimiterCalculator waits for all -// input streams to have at least one packet available before publishing. -TEST(RealTimeFlowLimiterCalculator, BasicTest) { - // Setup the calculator runner and add both ImageFrame and finish packets. - CalculatorRunner runner(GetDefaultNode()); - for (int i = 0; i < kNumImageFrames; ++i) { - Timestamp timestamp = Timestamp(i * Timestamp::kTimestampUnitsPerSecond); - runner.MutableInputs()->Index(0).packets.push_back( - MakePacket().At(timestamp)); - } - for (int i = 0; i < kNumFinished; ++i) { - Timestamp timestamp = - Timestamp((i + 1) * Timestamp::kTimestampUnitsPerSecond); - runner.MutableInputs() - ->Tag("FINISHED") - .packets.push_back(MakePacket(true).At(timestamp)); - } - - // Run the calculator. - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& frame_output_packets = - runner.Outputs().Index(0).packets; - - // Only outputs packets if both input streams are available. - int expected_num_packets = std::min(kNumImageFrames, kNumFinished + 1); - EXPECT_EQ(frame_output_packets.size(), expected_num_packets); -} - -// A Calculator::Process callback function. -typedef std::function - ProcessFunction; - -// A testing callback function that passes through all packets. -absl::Status PassthroughFunction(const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - for (int i = 0; i < inputs.NumEntries(); ++i) { - if (!inputs.Index(i).Value().IsEmpty()) { - outputs->Index(i).AddPacket(inputs.Index(i).Value()); - } - } - return absl::OkStatus(); -} - -// A Calculator that runs a testing callback function in Close. -class CloseCallbackCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - cc->Inputs().Get(id).SetAny(); - } - for (CollectionItemId id = cc->Outputs().BeginId(); - id < cc->Outputs().EndId(); ++id) { - cc->Outputs().Get(id).SetAny(); - } - cc->InputSidePackets().Index(0).Set>(); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return PassthroughFunction(cc->Inputs(), &(cc->Outputs())); - } - - absl::Status Close(CalculatorContext* cc) override { - const auto& callback = - cc->InputSidePackets().Index(0).Get>(); - return callback(); - } -}; -REGISTER_CALCULATOR(CloseCallbackCalculator); - -// Tests demostrating an RealTimeFlowLimiterCalculator operating in a cyclic -// graph. -// TODO: clean up these tests. -class RealTimeFlowLimiterCalculatorTest : public testing::Test { - public: - RealTimeFlowLimiterCalculatorTest() - : enter_semaphore_(0), exit_semaphore_(0) {} - - void SetUp() override { - graph_config_ = InflightGraphConfig(); - tool::AddVectorSink("out_1", &graph_config_, &out_1_packets_); - tool::AddVectorSink("out_2", &graph_config_, &out_2_packets_); - } - - void InitializeGraph(int max_in_flight) { - ProcessFunction semaphore_0_func = [&](const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - enter_semaphore_.Release(1); - return PassthroughFunction(inputs, outputs); - }; - ProcessFunction semaphore_1_func = [&](const InputStreamShardSet& inputs, - OutputStreamShardSet* outputs) { - exit_semaphore_.Acquire(1); - return PassthroughFunction(inputs, outputs); - }; - std::function close_func = [this]() { - close_count_++; - return absl::OkStatus(); - }; - MP_ASSERT_OK(graph_.Initialize( - graph_config_, { - {"max_in_flight", MakePacket(max_in_flight)}, - {"callback_0", Adopt(new auto(semaphore_0_func))}, - {"callback_1", Adopt(new auto(semaphore_1_func))}, - {"callback_2", Adopt(new auto(close_func))}, - })); - } - - // Adds a packet to a graph input stream. - void AddPacket(const std::string& input_name, int value) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(value).At(Timestamp(value)))); - } - - // A calculator graph starting with an RealTimeFlowLimiterCalculator and - // ending with a InFlightFinishCalculator. - // Back-edge "finished" limits processing to one frame in-flight. - // The two LambdaCalculators are used to keep certain packet sets in flight. - CalculatorGraphConfig InflightGraphConfig() { - return ParseTextProtoOrDie(R"pb( - input_stream: 'in_1' - input_stream: 'in_2' - node { - calculator: 'RealTimeFlowLimiterCalculator' - input_side_packet: 'MAX_IN_FLIGHT:max_in_flight' - input_stream: 'in_1' - input_stream: 'in_2' - input_stream: 'FINISHED:out_1' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_1_sampled' - output_stream: 'in_2_sampled' - } - node { - calculator: 'LambdaCalculator' - input_side_packet: 'callback_0' - input_stream: 'in_1_sampled' - input_stream: 'in_2_sampled' - output_stream: 'queue_1' - output_stream: 'queue_2' - } - node { - calculator: 'LambdaCalculator' - input_side_packet: 'callback_1' - input_stream: 'queue_1' - input_stream: 'queue_2' - output_stream: 'close_1' - output_stream: 'close_2' - } - node { - calculator: 'CloseCallbackCalculator' - input_side_packet: 'callback_2' - input_stream: 'close_1' - input_stream: 'close_2' - output_stream: 'out_1' - output_stream: 'out_2' - } - )pb"); - } - - protected: - CalculatorGraphConfig graph_config_; - CalculatorGraph graph_; - AtomicSemaphore enter_semaphore_; - AtomicSemaphore exit_semaphore_; - std::vector out_1_packets_; - std::vector out_2_packets_; - int close_count_ = 0; -}; - -// A test demonstrating an RealTimeFlowLimiterCalculator operating in a cyclic -// graph. This test shows that: -// -// (1) Timestamps are passed through unaltered. -// (2) All output streams including the back_edge stream are closed when -// the first input stream is closed. -// -TEST_F(RealTimeFlowLimiterCalculatorTest, BackEdgeCloses) { - InitializeGraph(1); - MP_ASSERT_OK(graph_.StartRun({})); - - auto send_packet = [this](const std::string& input_name, int64 n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - - for (int i = 0; i < 10; i++) { - send_packet("in_1", i * 10); - // This next input should be dropped. - send_packet("in_1", i * 10 + 5); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - send_packet("in_2", i * 10); - exit_semaphore_.Release(1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - } - MP_EXPECT_OK(graph_.CloseInputStream("in_1")); - MP_EXPECT_OK(graph_.CloseInputStream("in_2")); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - - // All output streams are closed and all output packets are delivered, - // with stream "in_1" and stream "in_2" closed. - EXPECT_EQ(10, out_1_packets_.size()); - EXPECT_EQ(10, out_2_packets_.size()); - - // Timestamps have not been messed with. - EXPECT_EQ(PacketValues(out_1_packets_), - TimestampValues(out_1_packets_)); - EXPECT_EQ(PacketValues(out_2_packets_), - TimestampValues(out_2_packets_)); - - // Extra inputs on in_1 have been dropped - EXPECT_EQ(TimestampValues(out_1_packets_), - (std::vector{0, 10, 20, 30, 40, 50, 60, 70, 80, 90})); - EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_)); - - // The closing of the stream has been propagated. - EXPECT_EQ(1, close_count_); -} - -// A test demonstrating that all output streams are closed when all -// input streams are closed after the last input packet has been processed. -TEST_F(RealTimeFlowLimiterCalculatorTest, AllStreamsClose) { - InitializeGraph(1); - MP_ASSERT_OK(graph_.StartRun({})); - - exit_semaphore_.Release(10); - for (int i = 0; i < 10; i++) { - AddPacket("in_1", i); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - AddPacket("in_2", i); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - } - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - - EXPECT_EQ(TimestampValues(out_1_packets_), TimestampValues(out_2_packets_)); - EXPECT_EQ(TimestampValues(out_1_packets_), - (std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); - EXPECT_EQ(1, close_count_); -} - -TEST(RealTimeFlowLimiterCalculator, TwoStreams) { - std::vector a_passed; - std::vector b_passed; - CalculatorGraphConfig graph_config_ = - ParseTextProtoOrDie(R"pb( - input_stream: 'in_a' - input_stream: 'in_b' - input_stream: 'finished' - node { - name: 'input_dropper' - calculator: 'RealTimeFlowLimiterCalculator' - input_side_packet: 'MAX_IN_FLIGHT:max_in_flight' - input_stream: 'in_a' - input_stream: 'in_b' - input_stream: 'FINISHED:finished' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_a_sampled' - output_stream: 'in_b_sampled' - output_stream: 'ALLOW:allow' - } - )pb"); - std::string allow_cb_name; - tool::AddVectorSink("in_a_sampled", &graph_config_, &a_passed); - tool::AddVectorSink("in_b_sampled", &graph_config_, &b_passed); - tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true); - - bool allow = true; - auto allow_cb = [&allow](const Packet& packet) { - allow = packet.Get(); - }; - - CalculatorGraph graph_; - MP_EXPECT_OK(graph_.Initialize( - graph_config_, - { - {"max_in_flight", MakePacket(1)}, - {allow_cb_name, - MakePacket>(allow_cb)}, - })); - - MP_EXPECT_OK(graph_.StartRun({})); - - auto send_packet = [&graph_](const std::string& input_name, int n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - send_packet("in_a", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(allow, false); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{})); - - send_packet("in_a", 2); - send_packet("in_b", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1})); - EXPECT_EQ(allow, false); - - send_packet("finished", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1})); - EXPECT_EQ(allow, true); - - send_packet("in_b", 2); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1})); - EXPECT_EQ(allow, true); - - send_packet("in_b", 3); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1, 3})); - EXPECT_EQ(allow, false); - - send_packet("in_b", 4); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1, 3})); - EXPECT_EQ(allow, false); - - send_packet("in_a", 3); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1, 3})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1, 3})); - EXPECT_EQ(allow, false); - - send_packet("finished", 3); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(TimestampValues(a_passed), (std::vector{1, 3})); - EXPECT_EQ(TimestampValues(b_passed), (std::vector{1, 3})); - EXPECT_EQ(allow, true); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -TEST(RealTimeFlowLimiterCalculator, CanConsume) { - std::vector in_sampled_packets_; - CalculatorGraphConfig graph_config_ = - ParseTextProtoOrDie(R"pb( - input_stream: 'in' - input_stream: 'finished' - node { - name: 'input_dropper' - calculator: 'RealTimeFlowLimiterCalculator' - input_side_packet: 'MAX_IN_FLIGHT:max_in_flight' - input_stream: 'in' - input_stream: 'FINISHED:finished' - input_stream_info: { tag_index: 'FINISHED' back_edge: true } - output_stream: 'in_sampled' - output_stream: 'ALLOW:allow' - } - )pb"); - std::string allow_cb_name; - tool::AddVectorSink("in_sampled", &graph_config_, &in_sampled_packets_); - tool::AddCallbackCalculator("allow", &graph_config_, &allow_cb_name, true); - - bool allow = true; - auto allow_cb = [&allow](const Packet& packet) { - allow = packet.Get(); - }; - - CalculatorGraph graph_; - MP_EXPECT_OK(graph_.Initialize( - graph_config_, - { - {"max_in_flight", MakePacket(1)}, - {allow_cb_name, - MakePacket>(allow_cb)}, - })); - - MP_EXPECT_OK(graph_.StartRun({})); - - auto send_packet = [&graph_](const std::string& input_name, int n) { - MP_EXPECT_OK(graph_.AddPacketToInputStream( - input_name, MakePacket(n).At(Timestamp(n)))); - }; - send_packet("in", 1); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - EXPECT_EQ(allow, false); - EXPECT_EQ(TimestampValues(in_sampled_packets_), (std::vector{1})); - - MP_EXPECT_OK(in_sampled_packets_[0].Consume()); - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -} // anonymous namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/round_robin_demux_calculator.cc b/mediapipe/calculators/core/round_robin_demux_calculator.cc deleted file mode 100644 index 8c93bba71..000000000 --- a/mediapipe/calculators/core/round_robin_demux_calculator.cc +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace api2 { - -// Forwards the input packet to one of the n output streams "OUTPUT:0", -// "OUTPUT:1", ..., in round robin fashion. The index of the selected output -// stream is emitted to the output stream "SELECT". If not needed, the -// output stream "SELECT" may be omitted. -// -// Designed to run graph bottlenecks in parallel and thus reduce graph -// processing latency by parallelizing. -// -// A simple example config is: -// -// node { -// calculator: "RoundRobinDemuxCalculator" -// input_stream: "signal" -// output_stream: "OUTPUT:0:signal0" -// output_stream: "OUTPUT:1:signal1" -// output_stream: "SELECT:select" -// } -// -// node { -// calculator: "SlowCalculator" -// input_stream: "signal0" -// output_stream: "output0" -// } -// -// node { -// calculator: "SlowCalculator" -// input_stream: "signal1" -// output_stream: "output1" -// } -// -// node { -// calculator: "MuxCalculator" -// input_stream: "INPUT:0:output0" -// input_stream: "INPUT:1:output1" -// input_stream: "SELECT:select" -// output_stream: "OUTPUT:output" -// input_stream_handler { -// input_stream_handler: "MuxInputStreamHandler" -// } -// } -// -// which is essentially running the following configuration in parallel with a -// concurrency level of two: -// -// node { -// calculator: "SlowCalculator" -// input_stream: "signal" -// output_stream: "output" -// } -// -// If SlowCalculator has more than one output stream, the user can group the -// output with MakePairCalculator, MakeVectorCalculator, or a similar variant to -// use it with MuxCalculator and later unpack, or can create new variants of -// MuxCalculator/MuxInputStreamHandler. -class RoundRobinDemuxCalculator : public Node { - public: - static constexpr Input kIn{""}; - static constexpr Output::Optional kSelect{"SELECT"}; - static constexpr Output>::Multiple kOut{"OUTPUT"}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kSelect, kOut); - - absl::Status Open(CalculatorContext* cc) override { - output_data_stream_index_ = 0; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - kOut(cc)[output_data_stream_index_].Send(kIn(cc).packet()); - if (kSelect(cc).IsConnected()) { - kSelect(cc).Send(output_data_stream_index_); - } - output_data_stream_index_ = - (output_data_stream_index_ + 1) % kOut(cc).Count(); - return absl::OkStatus(); - } - - private: - int output_data_stream_index_; -}; - -MEDIAPIPE_REGISTER_NODE(RoundRobinDemuxCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/sequence_shift_calculator.cc b/mediapipe/calculators/core/sequence_shift_calculator.cc deleted file mode 100644 index 66dbdef2e..000000000 --- a/mediapipe/calculators/core/sequence_shift_calculator.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" - -namespace mediapipe { -namespace api2 { - -// A Calculator that shifts the timestamps of packets along a stream. Packets on -// the input stream are output with a timestamp of the packet given by packet -// offset. That is, packet[i] is output with the timestamp of -// packet[i + packet_offset]. Packet offset can be either positive or negative. -// If packet_offset is -n, the first n packets will be dropped. If packet offset -// is n, the final n packets will be dropped. For example, with a packet_offset -// of -1, the first packet on the stream will be dropped, the second will be -// output with the timestamp of the first, the third with the timestamp of the -// second, and so on. -class SequenceShiftCalculator : public Node { - public: - static constexpr Input kIn{""}; - static constexpr SideInput::Optional kOffset{"PACKET_OFFSET"}; - static constexpr Output> kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOffset, kOut, TimestampChange::Arbitrary()); - - // Reads from options to set cache_size_ and packet_offset_. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // A positive offset means we want a packet to be output with the timestamp of - // a later packet. Stores packets waiting for their output timestamps and - // outputs a single packet when the cache fills. - void ProcessPositiveOffset(CalculatorContext* cc); - - // A negative offset means we want a packet to be output with the timestamp of - // an earlier packet. Stores timestamps waiting for the corresponding input - // packet and outputs a single packet when the cache fills. - void ProcessNegativeOffset(CalculatorContext* cc); - - // Storage for packets waiting to be output when packet_offset > 0. When cache - // is full, oldest packet is output with current timestamp. - std::deque packet_cache_; - - // Storage for previous timestamps used when packet_offset < 0. When cache is - // full, oldest timestamp is used for current packet. - std::deque timestamp_cache_; - - // Copied from corresponding field in options. - int packet_offset_; - // The number of packets or timestamps we need to store to output packet[i] at - // the timestamp of packet[i + packet_offset]; equal to abs(packet_offset). - int cache_size_; -}; -MEDIAPIPE_REGISTER_NODE(SequenceShiftCalculator); - -absl::Status SequenceShiftCalculator::Open(CalculatorContext* cc) { - packet_offset_ = kOffset(cc).GetOr( - cc->Options().packet_offset()); - cache_size_ = abs(packet_offset_); - // An offset of zero is a no-op, but someone might still request it. - if (packet_offset_ == 0) { - cc->Outputs().Index(0).SetOffset(0); - } - return absl::OkStatus(); -} - -absl::Status SequenceShiftCalculator::Process(CalculatorContext* cc) { - if (packet_offset_ > 0) { - ProcessPositiveOffset(cc); - } else if (packet_offset_ < 0) { - ProcessNegativeOffset(cc); - } else { - kOut(cc).Send(kIn(cc).packet()); - } - return absl::OkStatus(); -} - -void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) { - if (packet_cache_.size() >= cache_size_) { - // Ready to output oldest packet with current timestamp. - kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp())); - packet_cache_.pop_front(); - } - // Store current packet for later output. - packet_cache_.push_back(kIn(cc).packet()); -} - -void SequenceShiftCalculator::ProcessNegativeOffset(CalculatorContext* cc) { - if (timestamp_cache_.size() >= cache_size_) { - // Ready to output current packet with oldest timestamp. - kOut(cc).Send(kIn(cc).packet().At(timestamp_cache_.front())); - timestamp_cache_.pop_front(); - } - // Store current timestamp for use by a future packet. - timestamp_cache_.push_back(cc->InputTimestamp()); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/core/sequence_shift_calculator.proto b/mediapipe/calculators/core/sequence_shift_calculator.proto deleted file mode 100644 index cdcd284ca..000000000 --- a/mediapipe/calculators/core/sequence_shift_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -message SequenceShiftCalculatorOptions { - extend CalculatorOptions { - optional SequenceShiftCalculatorOptions ext = 107633927; - } - optional int32 packet_offset = 1 [default = -1]; -} diff --git a/mediapipe/calculators/core/sequence_shift_calculator_test.cc b/mediapipe/calculators/core/sequence_shift_calculator_test.cc deleted file mode 100644 index 23ad57225..000000000 --- a/mediapipe/calculators/core/sequence_shift_calculator_test.cc +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -namespace { - -// Adds packets containing integers equal to their original timestamp. -void AddPackets(CalculatorRunner* runner) { - for (int i = 0; i < 10; ++i) { - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(new int(i)).At(Timestamp(i))); - } -} - -// Zero shift is a no-op (output input[i] at timestamp[i]). Input and output -// streams should be identical. -TEST(SequenceShiftCalculatorTest, ZeroShift) { - CalculatorRunner runner( - "SequenceShiftCalculator", - "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 0 }", 1, - 1, 0); - AddPackets(&runner); - MP_ASSERT_OK(runner.Run()); - const std::vector& input_packets = - runner.MutableInputs()->Index(0).packets; - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(10, input_packets.size()); - ASSERT_EQ(input_packets.size(), output_packets.size()); - for (int i = 0; i < output_packets.size(); ++i) { - // Make sure the contents are as expected. - EXPECT_EQ(input_packets[i].Get(), output_packets[i].Get()); - EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp()); - } -} - -// Tests shifting by three packets, i.e., output input[i] with the timestamp of -// input[i + 3]. -TEST(SequenceShiftCalculatorTest, PositiveShift) { - CalculatorRunner runner( - "SequenceShiftCalculator", - "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: 3 }", 1, - 1, 0); - AddPackets(&runner); - MP_ASSERT_OK(runner.Run()); - const std::vector& input_packets = - runner.MutableInputs()->Index(0).packets; - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(10, input_packets.size()); - // input_packet[i] should be output with the timestamp of input_packet[i + 3]. - // The last 3 packets are dropped. - ASSERT_EQ(7, output_packets.size()); - for (int i = 0; i < output_packets.size(); ++i) { - // Make sure the contents are as expected. - EXPECT_EQ(input_packets[i].Get(), output_packets[i].Get()); - // Make sure the timestamps are shifted as expected. - EXPECT_EQ(input_packets[i + 3].Timestamp(), output_packets[i].Timestamp()); - } -} - -// Tests shifting by -2, i.e., output input[i] with timestamp[i - 2]. The first -// two packets should be dropped. -TEST(SequenceShiftCalculatorTest, NegativeShift) { - CalculatorRunner runner( - "SequenceShiftCalculator", - "[mediapipe.SequenceShiftCalculatorOptions.ext]: { packet_offset: -2 }", - 1, 1, 0); - AddPackets(&runner); - MP_ASSERT_OK(runner.Run()); - const std::vector& input_packets = - runner.MutableInputs()->Index(0).packets; - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(10, input_packets.size()); - // Input packet[i] should be output with the timestamp of input packet[i - 2]. - // The first two packets are dropped. This means timestamps match between - // input and output packets, but the data in the output packets come from - // input_packets[i + 2]. - ASSERT_EQ(8, output_packets.size()); - for (int i = 0; i < output_packets.size(); ++i) { - EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp()); - EXPECT_EQ(input_packets[i + 2].Get(), output_packets[i].Get()); - } -} - -// Tests using a side packet to specify the offset. Shifting by -2, i.e., -// output input[i] with timestamp[i - 2]. The first two packets should be -// dropped. -TEST(SequenceShiftCalculatorTest, SidePacketOffset) { - CalculatorGraphConfig::Node node; - node.set_calculator("SequenceShiftCalculator"); - node.add_input_stream("input"); - node.add_output_stream("output"); - node.add_input_side_packet("PACKET_OFFSET:packet_offset"); - - CalculatorRunner runner(node); - AddPackets(&runner); - runner.MutableSidePackets()->Tag("PACKET_OFFSET") = Adopt(new int(-2)); - MP_ASSERT_OK(runner.Run()); - const std::vector& input_packets = - runner.MutableInputs()->Index(0).packets; - const std::vector& output_packets = runner.Outputs().Index(0).packets; - ASSERT_EQ(10, input_packets.size()); - // Input packet[i] should be output with the timestamp of input packet[i - 2]. - // The first two packets are dropped. This means timestamps match between - // input and output packets, but the data in the output packets come from - // input_packets[i + 2]. - ASSERT_EQ(8, output_packets.size()); - for (int i = 0; i < output_packets.size(); ++i) { - EXPECT_EQ(input_packets[i].Timestamp(), output_packets[i].Timestamp()); - EXPECT_EQ(input_packets[i + 2].Get(), output_packets[i].Get()); - } -} - -} // namespace - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc deleted file mode 100644 index ed89889df..000000000 --- a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -using mediapipe::PacketTypeSet; -using mediapipe::Timestamp; - -namespace { - -constexpr char kTagAtPreStream[] = "AT_PRESTREAM"; -constexpr char kTagAtPostStream[] = "AT_POSTSTREAM"; -constexpr char kTagAtZero[] = "AT_ZERO"; -constexpr char kTagAtTick[] = "AT_TICK"; -constexpr char kTagTick[] = "TICK"; -constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP"; -constexpr char kTagSideInputTimestamp[] = "TIMESTAMP"; - -static std::map* kTimestampMap = []() { - auto* res = new std::map(); - res->emplace(kTagAtPreStream, Timestamp::PreStream()); - res->emplace(kTagAtPostStream, Timestamp::PostStream()); - res->emplace(kTagAtZero, Timestamp(0)); - res->emplace(kTagAtTick, Timestamp::Unset()); - res->emplace(kTagAtTimestamp, Timestamp::Unset()); - return res; -}(); - -template -std::string GetOutputTag(const CC& cc) { - // Single output tag only is required by contract. - return *cc.Outputs().GetTags().begin(); -} - -} // namespace - -// Outputs side packet(s) in corresponding output stream(s) with a particular -// timestamp, depending on the tag used to define output stream(s). (One tag can -// be used only.) -// -// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP -// and corresponding timestamps are Timestamp::PreStream(), -// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK -// input, and timestamp received from a side input. -// -// Examples: -// node { -// calculator: "SidePacketToStreamCalculator" -// input_side_packet: "side_packet" -// output_stream: "AT_PRESTREAM:packet" -// } -// -// node { -// calculator: "SidePacketToStreamCalculator" -// input_stream: "TICK:tick" -// input_side_packet: "side_packet" -// output_stream: "AT_TICK:packet" -// } -// -// node { -// calculator: "SidePacketToStreamCalculator" -// input_side_packet: "TIMESTAMP:timestamp" -// input_side_packet: "side_packet" -// output_stream: "AT_TIMESTAMP:packet" -// } -class SidePacketToStreamCalculator : public CalculatorBase { - public: - SidePacketToStreamCalculator() = default; - ~SidePacketToStreamCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - bool is_tick_processing_ = false; - std::string output_tag_; -}; -REGISTER_CALCULATOR(SidePacketToStreamCalculator); - -absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) { - const auto& tags = cc->Outputs().GetTags(); - RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1) - << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s)."; - RET_CHECK( - (cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) || - (!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick))) - << "Either both of TICK and AT_TICK should be used or none of them."; - RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) && - cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) || - (!cc->Outputs().HasTag(kTagAtTimestamp) && - !cc->InputSidePackets().HasTag(kTagSideInputTimestamp))) - << "Either both TIMESTAMP and AT_TIMESTAMP should be used or none of " - "them."; - const std::string output_tag = GetOutputTag(*cc); - const int num_entries = cc->Outputs().NumEntries(output_tag); - if (cc->Outputs().HasTag(kTagAtTimestamp)) { - RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries()) - << "For AT_TIMESTAMP tag, 2 input side packets are required."; - cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set(); - } else { - RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries()) - << "Same number of input side packets and output streams is required."; - } - for (int i = 0; i < num_entries; ++i) { - cc->InputSidePackets().Index(i).SetAny(); - cc->Outputs() - .Get(output_tag, i) - .SetSameAs(cc->InputSidePackets().Index(i).GetSameAs()); - } - - if (cc->Inputs().HasTag(kTagTick)) { - cc->Inputs().Tag(kTagTick).SetAny(); - } - - return absl::OkStatus(); -} - -absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) { - output_tag_ = GetOutputTag(*cc); - if (cc->Inputs().HasTag(kTagTick)) { - is_tick_processing_ = true; - // Set offset, so output timestamp bounds are updated in response to TICK - // timestamp bound update. - cc->SetOffset(TimestampDiff(0)); - } - return absl::OkStatus(); -} - -absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) { - if (is_tick_processing_) { - // TICK input is guaranteed to be non-empty, as it's the only input stream - // for this calculator. - const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp(); - for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { - cc->Outputs() - .Get(output_tag_, i) - .AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); - } - - return absl::OkStatus(); - } - - return mediapipe::tool::StatusStop(); -} - -absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) { - if (!cc->Outputs().HasTag(kTagAtTick) && - !cc->Outputs().HasTag(kTagAtTimestamp)) { - const auto& timestamp = kTimestampMap->at(output_tag_); - for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { - cc->Outputs() - .Get(output_tag_, i) - .AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); - } - } else if (cc->Outputs().HasTag(kTagAtTimestamp)) { - int64 timestamp = - cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get(); - for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { - cc->Outputs() - .Get(output_tag_, i) - .AddPacket(cc->InputSidePackets().Index(i).At(Timestamp(timestamp))); - } - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc deleted file mode 100644 index 22c27940e..000000000 --- a/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/match.h" -#include "absl/strings/str_replace.h" -#include "absl/strings/string_view.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/options_util.h" - -namespace mediapipe { -namespace { - -using testing::HasSubstr; - -TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "tick" - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet" - output_stream: "AT_TICK:packet" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr( - "Either both of TICK and AT_TICK should be used or none of them.")); -} - -TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "timestamp" - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet" - output_stream: "AT_TIMESTAMP:packet" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.message(), - HasSubstr("Either both TIMESTAMP and AT_TIMESTAMP should be used " - "or none of them.")); -} - -TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "tick" - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet" - output_stream: "DOES_NOT_EXIST:packet" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s).")); -} - -TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "tick" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "AT_TICK:packet0" - output_stream: "AT_PRE_STREAM:packet1" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s).")); -} - -TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet0" - output_stream: "AT_PRESTREAM:0:packet0" - output_stream: "AT_PRESTREAM:1:packet1" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr( - "Same number of input side packets and output streams is required.")); -} - -TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "AT_PRESTREAM:packet0" - } - )pb"); - CalculatorGraph graph; - auto status = graph.Initialize(graph_config); - EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr( - "Same number of input side packets and output streams is required.")); -} - -void DoTestNonAtTickOutputTag(absl::string_view tag, - Timestamp expected_timestamp) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(absl::StrReplaceAll( - R"( - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "side_packet" - output_stream: "$tag:packet" - } - )", - {{"$tag", tag}})); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - const int expected_value = 10; - std::vector output_packets; - MP_ASSERT_OK(graph.ObserveOutputStream( - "packet", [&output_packets](const Packet& packet) { - output_packets.push_back(packet); - return absl::OkStatus(); - })); - MP_ASSERT_OK( - graph.StartRun({{"side_packet", MakePacket(expected_value)}})); - MP_ASSERT_OK(graph.WaitForObservedOutput()); - - ASSERT_FALSE(output_packets.empty()); - EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp()); - EXPECT_EQ(expected_value, output_packets.back().Get()); -} - -TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) { - DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream()); - DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream()); - DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0)); -} - -TEST(SidePacketToStreamCalculator, AtTick) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "tick" - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_stream: "TICK:tick" - input_side_packet: "side_packet" - output_stream: "AT_TICK:packet" - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("packet", &graph_config, &output_packets); - CalculatorGraph graph; - - MP_ASSERT_OK(graph.Initialize(graph_config)); - const int expected_value = 20; - MP_ASSERT_OK( - graph.StartRun({{"side_packet", MakePacket(expected_value)}})); - - auto tick_and_verify = [&graph, &output_packets, - expected_value](int at_timestamp) { - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tick", - MakePacket(/*doesn't matter*/ 1).At(Timestamp(at_timestamp)))); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ASSERT_FALSE(output_packets.empty()); - EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp()); - EXPECT_EQ(expected_value, output_packets.back().Get()); - }; - - tick_and_verify(/*at_timestamp=*/0); - tick_and_verify(/*at_timestamp=*/1); - tick_and_verify(/*at_timestamp=*/128); - tick_and_verify(/*at_timestamp=*/1024); - tick_and_verify(/*at_timestamp=*/1025); -} - -TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_stream: "tick" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "packet0" - output_stream: "packet1" - node { - calculator: "SidePacketToStreamCalculator" - input_stream: "TICK:tick" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "AT_TICK:0:packet0" - output_stream: "AT_TICK:1:packet1" - } - )pb"); - std::vector output_packets0; - tool::AddVectorSink("packet0", &graph_config, &output_packets0); - std::vector output_packets1; - tool::AddVectorSink("packet1", &graph_config, &output_packets1); - CalculatorGraph graph; - - MP_ASSERT_OK(graph.Initialize(graph_config)); - const int expected_value0 = 20; - const int expected_value1 = 128; - MP_ASSERT_OK( - graph.StartRun({{"side_packet0", MakePacket(expected_value0)}, - {"side_packet1", MakePacket(expected_value1)}})); - - auto tick_and_verify = [&graph, &output_packets0, &output_packets1, - expected_value0, expected_value1](int at_timestamp) { - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tick", - MakePacket(/*doesn't matter*/ 1).At(Timestamp(at_timestamp)))); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ASSERT_FALSE(output_packets0.empty()); - ASSERT_FALSE(output_packets1.empty()); - - EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp()); - EXPECT_EQ(expected_value0, output_packets0.back().Get()); - EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp()); - EXPECT_EQ(expected_value1, output_packets1.back().Get()); - }; - - tick_and_verify(/*at_timestamp=*/0); - tick_and_verify(/*at_timestamp=*/1); - tick_and_verify(/*at_timestamp=*/128); - tick_and_verify(/*at_timestamp=*/1024); - tick_and_verify(/*at_timestamp=*/1025); -} - -TEST(SidePacketToStreamCalculator, AtTimestamp) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_side_packet: "timestamp" - input_side_packet: "side_packet" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "TIMESTAMP:timestamp" - input_side_packet: "side_packet" - output_stream: "AT_TIMESTAMP:packet" - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("packet", &graph_config, &output_packets); - CalculatorGraph graph; - - MP_ASSERT_OK(graph.Initialize(graph_config)); - const int expected_value = 20; - const int64 expected_timestamp = 5; - MP_ASSERT_OK( - graph.StartRun({{"side_packet", MakePacket(expected_value)}, - {"timestamp", MakePacket(expected_timestamp)}})); - - MP_ASSERT_OK(graph.WaitUntilDone()); - - ASSERT_FALSE(output_packets.empty()); - EXPECT_EQ(Timestamp(expected_timestamp), output_packets.back().Timestamp()); - EXPECT_EQ(expected_value, output_packets.back().Get()); -} - -TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) { - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie( - R"pb( - input_side_packet: "timestamp" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "packet" - node { - calculator: "SidePacketToStreamCalculator" - input_side_packet: "TIMESTAMP:timestamp" - input_side_packet: "side_packet0" - input_side_packet: "side_packet1" - output_stream: "AT_TIMESTAMP:0:packet0" - output_stream: "AT_TIMESTAMP:1:packet1" - } - )pb"); - std::vector output_packets0; - tool::AddVectorSink("packet0", &graph_config, &output_packets0); - std::vector output_packets1; - tool::AddVectorSink("packet1", &graph_config, &output_packets1); - CalculatorGraph graph; - - MP_ASSERT_OK(graph.Initialize(graph_config)); - const int expected_value0 = 20; - const int expected_value1 = 15; - const int64 expected_timestamp = 5; - MP_ASSERT_OK( - graph.StartRun({{"side_packet0", MakePacket(expected_value0)}, - {"side_packet1", MakePacket(expected_value1)}, - {"timestamp", MakePacket(expected_timestamp)}})); - - MP_ASSERT_OK(graph.WaitUntilDone()); - - ASSERT_FALSE(output_packets0.empty()); - EXPECT_EQ(Timestamp(expected_timestamp), output_packets0.back().Timestamp()); - EXPECT_EQ(expected_value0, output_packets0.back().Get()); - ASSERT_FALSE(output_packets1.empty()); - EXPECT_EQ(Timestamp(expected_timestamp), output_packets1.back().Timestamp()); - EXPECT_EQ(expected_value1, output_packets1.back().Get()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc b/mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc deleted file mode 100644 index d57cebe9c..000000000 --- a/mediapipe/calculators/core/split_normalized_landmark_list_calculator.cc +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ // NOLINT -#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ // NOLINT - -#include "mediapipe/calculators/core/split_vector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" - -namespace mediapipe { - -// Splits an input packet with NormalizedLandmarkList into -// multiple NormalizedLandmarkList output packets using the [begin, end) ranges -// specified in SplitVectorCalculatorOptions. If the option "element_only" is -// set to true, all ranges should be of size 1 and all outputs will be elements -// of type NormalizedLandmark. If "element_only" is false, ranges can be -// non-zero in size and all outputs will be of type NormalizedLandmarkList. -// If the option "combine_outputs" is set to true, only one output stream can be -// specified and all ranges of elements will be combined into one -// NormalizedLandmarkList. -class SplitNormalizedLandmarkListCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().NumEntries() == 1); - RET_CHECK(cc->Outputs().NumEntries() != 0); - - cc->Inputs().Index(0).Set(); - - const auto& options = - cc->Options<::mediapipe::SplitVectorCalculatorOptions>(); - - if (options.combine_outputs()) { - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1); - cc->Outputs().Index(0).Set(); - for (int i = 0; i < options.ranges_size() - 1; ++i) { - for (int j = i + 1; j < options.ranges_size(); ++j) { - const auto& range_0 = options.ranges(i); - const auto& range_1 = options.ranges(j); - if ((range_0.begin() >= range_1.begin() && - range_0.begin() < range_1.end()) || - (range_1.begin() >= range_0.begin() && - range_1.begin() < range_0.end())) { - return absl::InvalidArgumentError( - "Ranges must be non-overlapping when using combine_outputs " - "option."); - } - } - } - } else { - if (cc->Outputs().NumEntries() != options.ranges_size()) { - return absl::InvalidArgumentError( - "The number of output streams should match the number of ranges " - "specified in the CalculatorOptions."); - } - - // Set the output types for each output stream. - for (int i = 0; i < cc->Outputs().NumEntries(); ++i) { - if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 || - options.ranges(i).begin() >= options.ranges(i).end()) { - return absl::InvalidArgumentError( - "Indices should be non-negative and begin index should be less " - "than the end index."); - } - if (options.element_only()) { - if (options.ranges(i).end() - options.ranges(i).begin() != 1) { - return absl::InvalidArgumentError( - "Since element_only is true, all ranges should be of size 1."); - } - cc->Outputs().Index(i).Set(); - } else { - cc->Outputs().Index(i).Set(); - } - } - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = - cc->Options<::mediapipe::SplitVectorCalculatorOptions>(); - - element_only_ = options.element_only(); - combine_outputs_ = options.combine_outputs(); - - for (const auto& range : options.ranges()) { - ranges_.push_back({range.begin(), range.end()}); - max_range_end_ = std::max(max_range_end_, range.end()); - total_elements_ += range.end() - range.begin(); - } - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const NormalizedLandmarkList& input = - cc->Inputs().Index(0).Get(); - RET_CHECK_GE(input.landmark_size(), max_range_end_) - << "Max range end " << max_range_end_ << " exceeds landmarks size " - << input.landmark_size(); - - if (combine_outputs_) { - NormalizedLandmarkList output; - for (int i = 0; i < ranges_.size(); ++i) { - for (int j = ranges_[i].first; j < ranges_[i].second; ++j) { - const NormalizedLandmark& input_landmark = input.landmark(j); - *output.add_landmark() = input_landmark; - } - } - RET_CHECK_EQ(output.landmark_size(), total_elements_); - cc->Outputs().Index(0).AddPacket( - MakePacket(output).At(cc->InputTimestamp())); - } else { - if (element_only_) { - for (int i = 0; i < ranges_.size(); ++i) { - cc->Outputs().Index(i).AddPacket( - MakePacket(input.landmark(ranges_[i].first)) - .At(cc->InputTimestamp())); - } - } else { - for (int i = 0; i < ranges_.size(); ++i) { - NormalizedLandmarkList output; - for (int j = ranges_[i].first; j < ranges_[i].second; ++j) { - const NormalizedLandmark& input_landmark = input.landmark(j); - *output.add_landmark() = input_landmark; - } - cc->Outputs().Index(i).AddPacket( - MakePacket(output).At( - cc->InputTimestamp())); - } - } - } - - return absl::OkStatus(); - } - - private: - std::vector> ranges_; - int32 max_range_end_ = -1; - int32 total_elements_ = 0; - bool element_only_ = false; - bool combine_outputs_ = false; -}; - -REGISTER_CALCULATOR(SplitNormalizedLandmarkListCalculator); - -} // namespace mediapipe - -// NOLINTNEXTLINE -#endif // MEDIAPIPE_CALCULATORS_CORE_SPLIT_NORMALIZED_LANDMARK_LIST_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc b/mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc deleted file mode 100644 index bc1a028e0..000000000 --- a/mediapipe/calculators/core/split_normalized_landmark_list_calculator_test.cc +++ /dev/null @@ -1,404 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/calculators/core/split_vector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -constexpr float kLocationVal = 3; - -class SplitNormalizedLandmarkListCalculatorTest : public ::testing::Test { - protected: - void TearDown() { expected_landmarks_.reset(); } - - void PrepareNormalizedLandmarkList(int list_size) { - // Prepare input landmark list. - input_landmarks_ = absl::make_unique(); - expected_landmarks_ = absl::make_unique(); - for (int i = 0; i < list_size; ++i) { - NormalizedLandmark* landmark = input_landmarks_->add_landmark(); - landmark->set_x(i * kLocationVal); - landmark->set_y(i * kLocationVal); - landmark->set_z(i * kLocationVal); - // Save the landmarks for comparison after the graph runs. - *expected_landmarks_->add_landmark() = *landmark; - } - } - - void ValidateListOutput(std::vector& output_packets, - int expected_elements, int input_begin_index) { - ASSERT_EQ(1, output_packets.size()); - const NormalizedLandmarkList& output_landmarks = - output_packets[0].Get(); - ASSERT_EQ(expected_elements, output_landmarks.landmark_size()); - - for (int i = 0; i < expected_elements; ++i) { - const NormalizedLandmark& expected_landmark = - expected_landmarks_->landmark(input_begin_index + i); - const NormalizedLandmark& result = output_landmarks.landmark(i); - EXPECT_FLOAT_EQ(expected_landmark.x(), result.x()); - EXPECT_FLOAT_EQ(expected_landmark.y(), result.y()); - EXPECT_FLOAT_EQ(expected_landmark.z(), result.z()); - } - } - - void ValidateCombinedListOutput(std::vector& output_packets, - int expected_elements, - std::vector& input_begin_indices, - std::vector& input_end_indices) { - ASSERT_EQ(1, output_packets.size()); - ASSERT_EQ(input_begin_indices.size(), input_end_indices.size()); - const NormalizedLandmarkList& output_landmarks = - output_packets[0].Get(); - ASSERT_EQ(expected_elements, output_landmarks.landmark_size()); - const int num_ranges = input_begin_indices.size(); - - int element_id = 0; - for (int range_id = 0; range_id < num_ranges; ++range_id) { - for (int i = input_begin_indices[range_id]; - i < input_end_indices[range_id]; ++i) { - const NormalizedLandmark& expected_landmark = - expected_landmarks_->landmark(i); - const NormalizedLandmark& result = - output_landmarks.landmark(element_id); - EXPECT_FLOAT_EQ(expected_landmark.x(), result.x()); - EXPECT_FLOAT_EQ(expected_landmark.y(), result.y()); - EXPECT_FLOAT_EQ(expected_landmark.z(), result.z()); - element_id++; - } - } - } - - void ValidateElementOutput(std::vector& output_packets, - int input_begin_index) { - ASSERT_EQ(1, output_packets.size()); - - const NormalizedLandmark& output_landmark = - output_packets[0].Get(); - ASSERT_TRUE(output_landmark.IsInitialized()); - - const NormalizedLandmark& expected_landmark = - expected_landmarks_->landmark(input_begin_index); - - EXPECT_FLOAT_EQ(expected_landmark.x(), output_landmark.x()); - EXPECT_FLOAT_EQ(expected_landmark.y(), output_landmark.y()); - EXPECT_FLOAT_EQ(expected_landmark.z(), output_landmark.z()); - } - - std::unique_ptr input_landmarks_ = nullptr; - std::unique_ptr expected_landmarks_ = nullptr; - std::unique_ptr runner_ = nullptr; -}; - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTest) { - PrepareNormalizedLandmarkList(/*list_size=*/5); - ASSERT_NE(input_landmarks_, nullptr); - - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 4 } - ranges: { begin: 4 end: 5 } - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ValidateListOutput(range_0_packets, /*expected_elements=*/1, - /*input_begin_index=*/0); - ValidateListOutput(range_1_packets, /*expected_elements=*/3, - /*input_begin_index=*/1); - ValidateListOutput(range_2_packets, /*expected_elements=*/1, - /*input_begin_index=*/4); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("landmarks_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, InvalidRangeTest) { - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 0 } - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because of an invalid range (begin == end). - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, - InvalidOutputStreamCountTest) { - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - output_stream: "range_1" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because the number of output streams does not - // match the number of range elements in the options. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, - InvalidCombineOutputsMultipleOutputsTest) { - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - output_stream: "range_1" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - combine_outputs: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because the number of output streams does not - // match the number of range elements in the options. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, - InvalidOverlappingRangesTest) { - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 3 } - ranges: { begin: 1 end: 4 } - combine_outputs: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because there are overlapping ranges. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestElementOnly) { - PrepareNormalizedLandmarkList(/*list_size=*/5); - ASSERT_NE(input_landmarks_, nullptr); - - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - element_only: true - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ValidateElementOutput(range_0_packets, - /*input_begin_index=*/0); - ValidateElementOutput(range_1_packets, - /*input_begin_index=*/2); - ValidateElementOutput(range_2_packets, - /*input_begin_index=*/4); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("landmarks_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, SmokeTestCombiningOutputs) { - PrepareNormalizedLandmarkList(/*list_size=*/5); - ASSERT_NE(input_landmarks_, nullptr); - - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - combine_outputs: true - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "landmarks_in", Adopt(input_landmarks_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - std::vector input_begin_indices = {0, 2, 4}; - std::vector input_end_indices = {1, 3, 5}; - ValidateCombinedListOutput(range_0_packets, /*expected_elements=*/3, - input_begin_indices, input_end_indices); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("landmarks_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitNormalizedLandmarkListCalculatorTest, - ElementOnlyDisablesVectorOutputs) { - // Prepare a graph to use the SplitNormalizedLandmarkListCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "landmarks_in" - node { - calculator: "SplitNormalizedLandmarkListCalculator" - input_stream: "landmarks_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 4 } - ranges: { begin: 4 end: 5 } - element_only: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/split_vector_calculator.cc b/mediapipe/calculators/core/split_vector_calculator.cc deleted file mode 100644 index c8f1177d5..000000000 --- a/mediapipe/calculators/core/split_vector_calculator.cc +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/split_vector_calculator.h" - -#include - -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "tensorflow/lite/interpreter.h" - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - -namespace mediapipe { - -// Example config: -// node { -// calculator: "SplitTfLiteTensorVectorCalculator" -// input_stream: "tflitetensor_vector" -// output_stream: "tflitetensor_vector_range_0" -// output_stream: "tflitetensor_vector_range_1" -// options { -// [mediapipe.SplitVectorCalculatorOptions.ext] { -// ranges: { begin: 0 end: 1 } -// ranges: { begin: 1 end: 2 } -// element_only: false -// } -// } -// } -typedef SplitVectorCalculator - SplitTfLiteTensorVectorCalculator; -REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator); - -typedef SplitVectorCalculator SplitTensorVectorCalculator; -REGISTER_CALCULATOR(SplitTensorVectorCalculator); - -typedef SplitVectorCalculator - SplitLandmarkVectorCalculator; -REGISTER_CALCULATOR(SplitLandmarkVectorCalculator); - -typedef SplitVectorCalculator - SplitNormalizedLandmarkListVectorCalculator; -REGISTER_CALCULATOR(SplitNormalizedLandmarkListVectorCalculator); - -typedef SplitVectorCalculator - SplitNormalizedRectVectorCalculator; -REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator); - -typedef SplitVectorCalculator SplitMatrixVectorCalculator; -REGISTER_CALCULATOR(SplitMatrixVectorCalculator); - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true> - MovableSplitGlBufferVectorCalculator; -REGISTER_CALCULATOR(MovableSplitGlBufferVectorCalculator); -#endif - -typedef SplitVectorCalculator - SplitDetectionVectorCalculator; -REGISTER_CALCULATOR(SplitDetectionVectorCalculator); - -typedef SplitVectorCalculator - SplitClassificationListVectorCalculator; -REGISTER_CALCULATOR(SplitClassificationListVectorCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/split_vector_calculator.h b/mediapipe/calculators/core/split_vector_calculator.h deleted file mode 100644 index c77c6a40d..000000000 --- a/mediapipe/calculators/core/split_vector_calculator.h +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_ - -#include -#include - -#include "mediapipe/calculators/core/split_vector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" - -namespace mediapipe { - -template -using IsCopyable = std::enable_if_t::value, bool>; - -template -using IsNotCopyable = - std::enable_if_t::value, bool>; - -template -using IsMovable = std::enable_if_t::value, bool>; - -template -using IsNotMovable = - std::enable_if_t::value, bool>; - -// Splits an input packet with std::vector into multiple std::vector -// output packets using the [begin, end) ranges specified in -// SplitVectorCalculatorOptions. If the option "element_only" is set to true, -// all ranges should be of size 1 and all outputs will be elements of type T. If -// "element_only" is false, ranges can be non-zero in size and all outputs will -// be of type std::vector. If the option "combine_outputs" is set to true, -// only one output stream can be specified and all ranges of elements will be -// combined into one vector. -// To use this class for a particular type T, register a calculator using -// SplitVectorCalculator. -template -class SplitVectorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().NumEntries() == 1); - RET_CHECK(cc->Outputs().NumEntries() != 0); - - cc->Inputs().Index(0).Set>(); - - const auto& options = - cc->Options<::mediapipe::SplitVectorCalculatorOptions>(); - - if (!std::is_copy_constructible::value || move_elements) { - // Ranges of elements shouldn't overlap when the vector contains - // non-copyable elements. - RET_CHECK_OK(checkRangesDontOverlap(options)); - } - - if (options.combine_outputs()) { - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1); - cc->Outputs().Index(0).Set>(); - RET_CHECK_OK(checkRangesDontOverlap(options)); - } else { - if (cc->Outputs().NumEntries() != options.ranges_size()) { - return absl::InvalidArgumentError( - "The number of output streams should match the number of ranges " - "specified in the CalculatorOptions."); - } - - // Set the output types for each output stream. - for (int i = 0; i < cc->Outputs().NumEntries(); ++i) { - if (options.ranges(i).begin() < 0 || options.ranges(i).end() < 0 || - options.ranges(i).begin() >= options.ranges(i).end()) { - return absl::InvalidArgumentError( - "Indices should be non-negative and begin index should be less " - "than the end index."); - } - if (options.element_only()) { - if (options.ranges(i).end() - options.ranges(i).begin() != 1) { - return absl::InvalidArgumentError( - "Since element_only is true, all ranges should be of size 1."); - } - cc->Outputs().Index(i).Set(); - } else { - cc->Outputs().Index(i).Set>(); - } - } - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = - cc->Options<::mediapipe::SplitVectorCalculatorOptions>(); - - element_only_ = options.element_only(); - combine_outputs_ = options.combine_outputs(); - - for (const auto& range : options.ranges()) { - ranges_.push_back({range.begin(), range.end()}); - max_range_end_ = std::max(max_range_end_, range.end()); - total_elements_ += range.end() - range.begin(); - } - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (cc->Inputs().Index(0).IsEmpty()) return absl::OkStatus(); - - if (move_elements) { - return ProcessMovableElements(cc); - } else { - return ProcessCopyableElements(cc); - } - } - - template = true> - absl::Status ProcessCopyableElements(CalculatorContext* cc) { - // static_assert(std::is_copy_constructible::value, - // "Cannot copy non-copyable elements"); - const auto& input = cc->Inputs().Index(0).Get>(); - RET_CHECK_GE(input.size(), max_range_end_); - if (combine_outputs_) { - auto output = absl::make_unique>(); - output->reserve(total_elements_); - for (int i = 0; i < ranges_.size(); ++i) { - auto elements = absl::make_unique>( - input.begin() + ranges_[i].first, - input.begin() + ranges_[i].second); - output->insert(output->end(), elements->begin(), elements->end()); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else { - if (element_only_) { - for (int i = 0; i < ranges_.size(); ++i) { - cc->Outputs().Index(i).AddPacket( - MakePacket(input[ranges_[i].first]).At(cc->InputTimestamp())); - } - } else { - for (int i = 0; i < ranges_.size(); ++i) { - auto output = absl::make_unique>( - input.begin() + ranges_[i].first, - input.begin() + ranges_[i].second); - cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp()); - } - } - } - - return absl::OkStatus(); - } - - template = true> - absl::Status ProcessCopyableElements(CalculatorContext* cc) { - return absl::InternalError("Cannot copy non-copyable elements."); - } - - template = true> - absl::Status ProcessMovableElements(CalculatorContext* cc) { - absl::StatusOr>> input_status = - cc->Inputs().Index(0).Value().Consume>(); - if (!input_status.ok()) return input_status.status(); - std::unique_ptr> input_vector = - std::move(input_status).value(); - RET_CHECK_GE(input_vector->size(), max_range_end_); - - if (combine_outputs_) { - auto output = absl::make_unique>(); - output->reserve(total_elements_); - for (int i = 0; i < ranges_.size(); ++i) { - output->insert( - output->end(), - std::make_move_iterator(input_vector->begin() + ranges_[i].first), - std::make_move_iterator(input_vector->begin() + ranges_[i].second)); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else { - if (element_only_) { - for (int i = 0; i < ranges_.size(); ++i) { - cc->Outputs().Index(i).AddPacket( - MakePacket(std::move(input_vector->at(ranges_[i].first))) - .At(cc->InputTimestamp())); - } - } else { - for (int i = 0; i < ranges_.size(); ++i) { - auto output = absl::make_unique>(); - output->insert( - output->end(), - std::make_move_iterator(input_vector->begin() + ranges_[i].first), - std::make_move_iterator(input_vector->begin() + - ranges_[i].second)); - cc->Outputs().Index(i).Add(output.release(), cc->InputTimestamp()); - } - } - } - - return absl::OkStatus(); - } - - template = true> - absl::Status ProcessMovableElements(CalculatorContext* cc) { - return absl::InternalError("Cannot move non-movable elements."); - } - - private: - static absl::Status checkRangesDontOverlap( - const ::mediapipe::SplitVectorCalculatorOptions& options) { - for (int i = 0; i < options.ranges_size() - 1; ++i) { - for (int j = i + 1; j < options.ranges_size(); ++j) { - const auto& range_0 = options.ranges(i); - const auto& range_1 = options.ranges(j); - if ((range_0.begin() >= range_1.begin() && - range_0.begin() < range_1.end()) || - (range_1.begin() >= range_0.begin() && - range_1.begin() < range_0.end())) { - return absl::InvalidArgumentError( - "Ranges must be non-overlapping when using combine_outputs " - "option."); - } - } - } - return absl::OkStatus(); - } - - std::vector> ranges_; - int32 max_range_end_ = -1; - int32 total_elements_ = 0; - bool element_only_ = false; - bool combine_outputs_ = false; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_CORE_SPLIT_VECTOR_CALCULATOR_H_ diff --git a/mediapipe/calculators/core/split_vector_calculator.proto b/mediapipe/calculators/core/split_vector_calculator.proto deleted file mode 100644 index 40301f88b..000000000 --- a/mediapipe/calculators/core/split_vector_calculator.proto +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -option objc_class_prefix = "MediaPipe"; - -// A Range {begin, end} specifies beginning ane ending indices to splice a -// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with -// begin index inclusive and end index exclusive. -message Range { - optional int32 begin = 1; - optional int32 end = 2; -} - -message SplitVectorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional SplitVectorCalculatorOptions ext = 259438222; - } - - repeated Range ranges = 1; - - // Specify if single element ranges should be outputted as std::vector or - // just element of type T. By default, if a range specifies only one element, - // it is outputted as an std::vector. - optional bool element_only = 2 [default = false]; - - // Combines output elements to one vector. - optional bool combine_outputs = 3 [default = false]; -} diff --git a/mediapipe/calculators/core/split_vector_calculator_test.cc b/mediapipe/calculators/core/split_vector_calculator_test.cc deleted file mode 100644 index f68d98eb2..000000000 --- a/mediapipe/calculators/core/split_vector_calculator_test.cc +++ /dev/null @@ -1,694 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/core/split_vector_calculator.h" - -#include -#include -#include - -#include "mediapipe/calculators/core/split_vector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" - -namespace mediapipe { - -using ::tflite::Interpreter; - -const int width = 1; -const int height = 1; -const int channels = 1; - -class SplitTfLiteTensorVectorCalculatorTest : public ::testing::Test { - protected: - void TearDown() { - // Note: Since the pointers contained in this vector will be cleaned up by - // the interpreter, only ensure that the vector is cleaned up for the next - // test. - input_buffers_.clear(); - } - - void PrepareTfLiteTensorVector(int vector_size) { - ASSERT_NE(interpreter_, nullptr); - - // Prepare input tensors. - std::vector indices(vector_size); - for (int i = 0; i < vector_size; ++i) { - indices[i] = i; - } - interpreter_->AddTensors(vector_size); - interpreter_->SetInputs(indices); - - input_vec_ = absl::make_unique>(); - for (int i = 0; i < vector_size; ++i) { - interpreter_->SetTensorParametersReadWrite(i, kTfLiteFloat32, "", {3}, - TfLiteQuantization()); - const int tensor_index = interpreter_->inputs()[i]; - interpreter_->ResizeInputTensor(tensor_index, {width, height, channels}); - } - - interpreter_->AllocateTensors(); - - // Save the tensor buffer pointers for comparison after the graph runs. - input_buffers_ = std::vector(vector_size); - for (int i = 0; i < vector_size; ++i) { - const int tensor_index = interpreter_->inputs()[i]; - TfLiteTensor* tensor = interpreter_->tensor(tensor_index); - float* tensor_buffer = tensor->data.f; - ASSERT_NE(tensor_buffer, nullptr); - for (int j = 0; j < width * height * channels; ++j) { - tensor_buffer[j] = i; - } - input_vec_->push_back(*tensor); - input_buffers_[i] = tensor_buffer; - } - } - - void ValidateVectorOutput(std::vector& output_packets, - int expected_elements, int input_begin_index) { - ASSERT_EQ(1, output_packets.size()); - const std::vector& output_vec = - output_packets[0].Get>(); - ASSERT_EQ(expected_elements, output_vec.size()); - - for (int i = 0; i < expected_elements; ++i) { - const int expected_value = input_begin_index + i; - const TfLiteTensor* result = &output_vec[i]; - float* result_buffer = result->data.f; - ASSERT_NE(result_buffer, nullptr); - ASSERT_EQ(result_buffer, input_buffers_[input_begin_index + i]); - for (int j = 0; j < width * height * channels; ++j) { - ASSERT_EQ(expected_value, result_buffer[j]); - } - } - } - - void ValidateCombinedVectorOutput(std::vector& output_packets, - int expected_elements, - std::vector& input_begin_indices, - std::vector& input_end_indices) { - ASSERT_EQ(1, output_packets.size()); - ASSERT_EQ(input_begin_indices.size(), input_end_indices.size()); - const std::vector& output_vec = - output_packets[0].Get>(); - ASSERT_EQ(expected_elements, output_vec.size()); - const int num_ranges = input_begin_indices.size(); - - int element_id = 0; - for (int range_id = 0; range_id < num_ranges; ++range_id) { - for (int i = input_begin_indices[range_id]; - i < input_end_indices[range_id]; ++i) { - const int expected_value = i; - const TfLiteTensor* result = &output_vec[element_id]; - float* result_buffer = result->data.f; - ASSERT_NE(result_buffer, nullptr); - ASSERT_EQ(result_buffer, input_buffers_[i]); - for (int j = 0; j < width * height * channels; ++j) { - ASSERT_EQ(expected_value, result_buffer[j]); - } - element_id++; - } - } - } - - void ValidateElementOutput(std::vector& output_packets, - int input_begin_index) { - ASSERT_EQ(1, output_packets.size()); - - const TfLiteTensor& result = output_packets[0].Get(); - float* result_buffer = result.data.f; - ASSERT_NE(result_buffer, nullptr); - ASSERT_EQ(result_buffer, input_buffers_[input_begin_index]); - - const int expected_value = input_begin_index; - for (int j = 0; j < width * height * channels; ++j) { - ASSERT_EQ(expected_value, result_buffer[j]); - } - } - - std::unique_ptr interpreter_ = absl::make_unique(); - std::unique_ptr> input_vec_ = nullptr; - std::vector input_buffers_; - std::unique_ptr runner_ = nullptr; -}; - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTest) { - ASSERT_NE(interpreter_, nullptr); - - PrepareTfLiteTensorVector(/*vector_size=*/5); - ASSERT_NE(input_vec_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 4 } - ranges: { begin: 4 end: 5 } - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ValidateVectorOutput(range_0_packets, /*expected_elements=*/1, - /*input_begin_index=*/0); - ValidateVectorOutput(range_1_packets, /*expected_elements=*/3, - /*input_begin_index=*/1); - ValidateVectorOutput(range_2_packets, /*expected_elements=*/1, - /*input_begin_index=*/4); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidRangeTest) { - ASSERT_NE(interpreter_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 0 } - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because of an invalid range (begin == end). - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOutputStreamCountTest) { - ASSERT_NE(interpreter_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - output_stream: "range_1" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because the number of output streams does not - // match the number of range elements in the options. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, - InvalidCombineOutputsMultipleOutputsTest) { - ASSERT_NE(interpreter_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - output_stream: "range_1" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - combine_outputs: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because the number of output streams does not - // match the number of range elements in the options. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, InvalidOverlappingRangesTest) { - ASSERT_NE(interpreter_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 3 } - ranges: { begin: 1 end: 4 } - combine_outputs: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because there are overlapping ranges. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestElementOnly) { - ASSERT_NE(interpreter_, nullptr); - - PrepareTfLiteTensorVector(/*vector_size=*/5); - ASSERT_NE(input_vec_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - element_only: true - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - ValidateElementOutput(range_0_packets, - /*input_begin_index=*/0); - ValidateElementOutput(range_1_packets, - /*input_begin_index=*/2); - ValidateElementOutput(range_2_packets, - /*input_begin_index=*/4); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, SmokeTestCombiningOutputs) { - ASSERT_NE(interpreter_, nullptr); - - PrepareTfLiteTensorVector(/*vector_size=*/5); - ASSERT_NE(input_vec_, nullptr); - - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - combine_outputs: true - } - } - } - )pb"); - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec_.release()).At(Timestamp(0)))); - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - std::vector input_begin_indices = {0, 2, 4}; - std::vector input_end_indices = {1, 3, 5}; - ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3, - input_begin_indices, input_end_indices); - - // Fully close the graph at the end. - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(SplitTfLiteTensorVectorCalculatorTest, - ElementOnlyDisablesVectorOutputs) { - // Prepare a graph to use the SplitTfLiteTensorVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "tensor_in" - node { - calculator: "SplitTfLiteTensorVectorCalculator" - input_stream: "tensor_in" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 4 } - ranges: { begin: 4 end: 5 } - element_only: true - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -typedef SplitVectorCalculator, true> - MovableSplitUniqueIntPtrCalculator; -REGISTER_CALCULATOR(MovableSplitUniqueIntPtrCalculator); - -class MovableSplitUniqueIntPtrCalculatorTest : public ::testing::Test { - protected: - void ValidateVectorOutput(std::vector& output_packets, - int expected_elements, int input_begin_index) { - ASSERT_EQ(1, output_packets.size()); - const std::vector>& output_vec = - output_packets[0].Get>>(); - ASSERT_EQ(expected_elements, output_vec.size()); - - for (int i = 0; i < expected_elements; ++i) { - const int expected_value = input_begin_index + i; - const std::unique_ptr& result = output_vec[i]; - ASSERT_NE(result, nullptr); - ASSERT_EQ(expected_value, *result); - } - } - - void ValidateElementOutput(std::vector& output_packets, - int expected_value) { - ASSERT_EQ(1, output_packets.size()); - const std::unique_ptr& result = - output_packets[0].Get>(); - ASSERT_NE(result, nullptr); - ASSERT_EQ(expected_value, *result); - } - - void ValidateCombinedVectorOutput(std::vector& output_packets, - int expected_elements, - std::vector& input_begin_indices, - std::vector& input_end_indices) { - ASSERT_EQ(1, output_packets.size()); - ASSERT_EQ(input_begin_indices.size(), input_end_indices.size()); - const std::vector>& output_vector = - output_packets[0].Get>>(); - ASSERT_EQ(expected_elements, output_vector.size()); - const int num_ranges = input_begin_indices.size(); - - int element_id = 0; - for (int range_id = 0; range_id < num_ranges; ++range_id) { - for (int i = input_begin_indices[range_id]; - i < input_end_indices[range_id]; ++i) { - const int expected_value = i; - const std::unique_ptr& result = output_vector[element_id]; - ASSERT_NE(result, nullptr); - ASSERT_EQ(expected_value, *result); - ++element_id; - } - } - } -}; - -TEST_F(MovableSplitUniqueIntPtrCalculatorTest, InvalidOverlappingRangesTest) { - // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "input_vector" - node { - calculator: "MovableSplitUniqueIntPtrCalculator" - input_stream: "input_vector" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 3 } - ranges: { begin: 1 end: 4 } - } - } - } - )pb"); - - // Run the graph. - CalculatorGraph graph; - // The graph should fail running because there are overlapping ranges. - ASSERT_FALSE(graph.Initialize(graph_config).ok()); -} - -TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTest) { - // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "input_vector" - node { - calculator: "MovableSplitUniqueIntPtrCalculator" - input_stream: "input_vector" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 1 end: 4 } - ranges: { begin: 4 end: 5 } - } - } - } - )pb"); - - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - - // input_vector : {0, 1, 2, 3, 4, 5} - std::unique_ptr>> input_vector = - absl::make_unique>>(6); - for (int i = 0; i < 6; ++i) { - input_vector->at(i) = absl::make_unique(i); - } - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_vector", Adopt(input_vector.release()).At(Timestamp(1)))); - - MP_ASSERT_OK(graph.WaitUntilIdle()); - MP_ASSERT_OK(graph.CloseAllPacketSources()); - MP_ASSERT_OK(graph.WaitUntilDone()); - - ValidateVectorOutput(range_0_packets, /*expected_elements=*/1, - /*input_begin_index=*/0); - ValidateVectorOutput(range_1_packets, /*expected_elements=*/3, - /*input_begin_index=*/1); - ValidateVectorOutput(range_2_packets, /*expected_elements=*/1, - /*input_begin_index=*/4); -} - -TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestElementOnly) { - // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "input_vector" - node { - calculator: "MovableSplitUniqueIntPtrCalculator" - input_stream: "input_vector" - output_stream: "range_0" - output_stream: "range_1" - output_stream: "range_2" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - element_only: true - } - } - } - )pb"); - - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - std::vector range_1_packets; - tool::AddVectorSink("range_1", &graph_config, &range_1_packets); - std::vector range_2_packets; - tool::AddVectorSink("range_2", &graph_config, &range_2_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - - // input_vector : {0, 1, 2, 3, 4, 5} - std::unique_ptr>> input_vector = - absl::make_unique>>(6); - for (int i = 0; i < 6; ++i) { - input_vector->at(i) = absl::make_unique(i); - } - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_vector", Adopt(input_vector.release()).At(Timestamp(1)))); - - MP_ASSERT_OK(graph.WaitUntilIdle()); - MP_ASSERT_OK(graph.CloseAllPacketSources()); - MP_ASSERT_OK(graph.WaitUntilDone()); - - ValidateElementOutput(range_0_packets, /*expected_value=*/0); - ValidateElementOutput(range_1_packets, /*expected_value=*/2); - ValidateElementOutput(range_2_packets, /*expected_value=*/4); -} - -TEST_F(MovableSplitUniqueIntPtrCalculatorTest, SmokeTestCombiningOutputs) { - // Prepare a graph to use the TestMovableSplitUniqueIntPtrVectorCalculator. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - R"pb( - input_stream: "input_vector" - node { - calculator: "MovableSplitUniqueIntPtrCalculator" - input_stream: "input_vector" - output_stream: "range_0" - options { - [mediapipe.SplitVectorCalculatorOptions.ext] { - ranges: { begin: 0 end: 1 } - ranges: { begin: 2 end: 3 } - ranges: { begin: 4 end: 5 } - combine_outputs: true - } - } - } - )pb"); - - std::vector range_0_packets; - tool::AddVectorSink("range_0", &graph_config, &range_0_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - - // input_vector : {0, 1, 2, 3, 4, 5} - std::unique_ptr>> input_vector = - absl::make_unique>>(6); - for (int i = 0; i < 6; ++i) { - input_vector->at(i) = absl::make_unique(i); - } - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_vector", Adopt(input_vector.release()).At(Timestamp(1)))); - - MP_ASSERT_OK(graph.WaitUntilIdle()); - MP_ASSERT_OK(graph.CloseAllPacketSources()); - MP_ASSERT_OK(graph.WaitUntilDone()); - - std::vector input_begin_indices = {0, 2, 4}; - std::vector input_end_indices = {1, 3, 5}; - ValidateCombinedVectorOutput(range_0_packets, /*expected_elements=*/3, - input_begin_indices, input_end_indices); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/stream_to_side_packet_calculator.cc b/mediapipe/calculators/core/stream_to_side_packet_calculator.cc deleted file mode 100644 index 9dc25142a..000000000 --- a/mediapipe/calculators/core/stream_to_side_packet_calculator.cc +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { - -// A calculator that takes a packet of an input stream and converts it to an -// output side packet. This calculator only works under the assumption that the -// input stream only has a single packet passing through. -// -// Example config: -// node { -// calculator: "StreamToSidePacketCalculator" -// input_stream: "stream" -// output_side_packet: "side_packet" -// } -class StreamToSidePacketCalculator : public mediapipe::CalculatorBase { - public: - static absl::Status GetContract(mediapipe::CalculatorContract* cc) { - cc->Inputs().Index(0).SetAny(); - cc->OutputSidePackets().Index(0).SetAny(); - return absl::OkStatus(); - } - - absl::Status Process(mediapipe::CalculatorContext* cc) override { - mediapipe::Packet& packet = cc->Inputs().Index(0).Value(); - cc->OutputSidePackets().Index(0).Set( - packet.At(mediapipe::Timestamp::Unset())); - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(StreamToSidePacketCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/stream_to_side_packet_calculator_test.cc b/mediapipe/calculators/core/stream_to_side_packet_calculator_test.cc deleted file mode 100644 index 606f0e352..000000000 --- a/mediapipe/calculators/core/stream_to_side_packet_calculator_test.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { - -using ::testing::Test; - -class StreamToSidePacketCalculatorTest : public Test { - protected: - StreamToSidePacketCalculatorTest() { - const char kConfig[] = R"( - calculator: "StreamToSidePacketCalculator" - input_stream: "stream" - output_side_packet: "side_packet" - )"; - runner_ = absl::make_unique(kConfig); - } - - std::unique_ptr runner_; -}; - -TEST_F(StreamToSidePacketCalculatorTest, - StreamToSidePacketCalculatorWithEmptyStreamFails) { - EXPECT_EQ(runner_->Run().code(), absl::StatusCode::kUnavailable); -} - -TEST_F(StreamToSidePacketCalculatorTest, - StreamToSidePacketCalculatorWithSinglePacketCreatesSidePacket) { - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string("test")).At(Timestamp(1))); - MP_ASSERT_OK(runner_->Run()); - EXPECT_EQ(runner_->OutputSidePackets().Index(0).Get(), "test"); -} - -TEST_F(StreamToSidePacketCalculatorTest, - StreamToSidePacketCalculatorWithMultiplePacketsFails) { - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string("test1")).At(Timestamp(1))); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string("test2")).At(Timestamp(2))); - EXPECT_EQ(runner_->Run().code(), absl::StatusCode::kAlreadyExists); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/core/string_to_int_calculator.cc b/mediapipe/calculators/core/string_to_int_calculator.cc deleted file mode 100644 index 13a9a29e0..000000000 --- a/mediapipe/calculators/core/string_to_int_calculator.cc +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - -#include "absl/strings/numbers.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Calculator that converts a std::string into an integer type, or fails if the -// conversion is not possible. -// -// Example config: -// node { -// calculator: "StringToIntCalculator" -// input_side_packet: "string" -// output_side_packet: "index" -// } -template -class StringToIntCalculatorTemplate : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Index(0).Set(); - cc->OutputSidePackets().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - IntType number; - if (!absl::SimpleAtoi(cc->InputSidePackets().Index(0).Get(), - &number)) { - return absl::InvalidArgumentError( - "The std::string could not be parsed as an integer."); - } - cc->OutputSidePackets().Index(0).Set(MakePacket(number)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; - -using StringToIntCalculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToIntCalculator); - -using StringToUintCalculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToUintCalculator); - -using StringToInt32Calculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToInt32Calculator); - -using StringToUint32Calculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToUint32Calculator); - -using StringToInt64Calculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToInt64Calculator); - -using StringToUint64Calculator = StringToIntCalculatorTemplate; -REGISTER_CALCULATOR(StringToUint64Calculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD deleted file mode 100644 index 39f81c046..000000000 --- a/mediapipe/calculators/image/BUILD +++ /dev/null @@ -1,604 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -mediapipe_proto_library( - name = "opencv_image_encoder_calculator_proto", - srcs = ["opencv_image_encoder_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "scale_image_calculator_proto", - srcs = ["scale_image_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/framework/formats:image_format_proto", - ], -) - -mediapipe_proto_library( - name = "set_alpha_calculator_proto", - srcs = ["set_alpha_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "image_cropping_calculator_proto", - srcs = ["image_cropping_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "bilateral_filter_calculator_proto", - srcs = ["bilateral_filter_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "recolor_calculator_proto", - srcs = ["recolor_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - ], -) - -cc_library( - name = "color_convert_calculator", - srcs = ["color_convert_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "opencv_encoded_image_to_image_frame_calculator", - srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":opencv_encoded_image_to_image_frame_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "opencv_image_encoder_calculator", - srcs = ["opencv_image_encoder_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":opencv_image_encoder_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "opencv_put_text_calculator", - srcs = ["opencv_put_text_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "set_alpha_calculator", - srcs = ["set_alpha_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":set_alpha_calculator_cc_proto", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gl_quad_renderer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "bilateral_filter_calculator", - srcs = ["bilateral_filter_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":bilateral_filter_calculator_cc_proto", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gl_quad_renderer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -mediapipe_proto_library( - name = "image_transformation_calculator_proto", - srcs = ["image_transformation_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/gpu:scale_mode_proto", - ], -) - -cc_library( - name = "image_transformation_calculator", - srcs = ["image_transformation_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:apple": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":image_transformation_calculator_cc_proto", - "//mediapipe/gpu:scale_mode_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gl_quad_renderer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "image_cropping_calculator", - srcs = ["image_cropping_calculator.cc"], - hdrs = ["image_cropping_calculator.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:apple": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":image_cropping_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gl_quad_renderer", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -cc_test( - name = "image_cropping_calculator_test", - srcs = ["image_cropping_calculator_test.cc"], - deps = [ - ":image_cropping_calculator", - ":image_cropping_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map", - "//mediapipe/framework/tool:tag_map_helper", - ], -) - -cc_library( - name = "luminance_calculator", - srcs = ["luminance_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/gpu:gl_simple_calculator", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:shader_util", - ], - alwayslink = 1, -) - -cc_library( - name = "sobel_edges_calculator", - srcs = ["sobel_edges_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/gpu:gl_simple_calculator", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:shader_util", - ], - alwayslink = 1, -) - -cc_library( - name = "recolor_calculator", - srcs = ["recolor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":recolor_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gl_quad_renderer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "scale_image_utils", - srcs = ["scale_image_utils.cc"], - hdrs = ["scale_image_utils.h"], - visibility = [ - "//mediapipe:__subpackages__", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "scale_image_calculator", - srcs = ["scale_image_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":scale_image_utils", - "//mediapipe/calculators/image:scale_image_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/formats:yuv_image", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:image_resizer", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:image_frame_util", - "@com_google_absl//absl/strings", - "@libyuv", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "image_clone_calculator_proto", - srcs = ["image_clone_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "image_clone_calculator", - srcs = ["image_clone_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":image_clone_calculator_cc_proto", - "//mediapipe/framework/api2:node", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "image_properties_calculator", - srcs = ["image_properties_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/api2:node", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gpu_buffer", - ], - }), - alwayslink = 1, -) - -cc_test( - name = "opencv_encoded_image_to_image_frame_calculator_test", - srcs = ["opencv_encoded_image_to_image_frame_calculator_test.cc"], - data = ["//mediapipe/calculators/image/testdata:test_images"], - deps = [ - ":opencv_encoded_image_to_image_frame_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_test( - name = "opencv_image_encoder_calculator_test", - srcs = ["opencv_image_encoder_calculator_test.cc"], - data = ["//mediapipe/calculators/image/testdata:test_images"], - deps = [ - ":opencv_image_encoder_calculator", - ":opencv_image_encoder_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_test( - name = "scale_image_utils_test", - srcs = ["scale_image_utils_test.cc"], - deps = [ - ":scale_image_utils", - "//mediapipe/framework/port:gtest_main", - ], -) - -mediapipe_proto_library( - name = "mask_overlay_calculator_proto", - srcs = ["mask_overlay_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "opencv_encoded_image_to_image_frame_calculator_proto", - srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "feature_detector_calculator_proto", - srcs = ["feature_detector_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "mask_overlay_calculator", - srcs = ["mask_overlay_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":mask_overlay_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:shader_util", - ], - alwayslink = 1, -) - -cc_library( - name = "feature_detector_calculator", - srcs = ["feature_detector_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":feature_detector_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_features2d", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:threadpool", - "//mediapipe/framework/tool:options_util", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/synchronization", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "image_file_properties_calculator", - srcs = ["image_file_properties_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_file_properties_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@easyexif", - ], - alwayslink = 1, -) - -cc_test( - name = "image_file_properties_calculator_test", - srcs = ["image_file_properties_calculator_test.cc"], - data = ["//mediapipe/calculators/image/testdata:test_images"], - deps = [ - ":image_file_properties_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_file_properties_cc_proto", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) diff --git a/mediapipe/calculators/image/bilateral_filter_calculator.cc b/mediapipe/calculators/image/bilateral_filter_calculator.cc deleted file mode 100644 index 3d878bffc..000000000 --- a/mediapipe/calculators/image/bilateral_filter_calculator.cc +++ /dev/null @@ -1,518 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { - -namespace { -constexpr char kInputFrameTag[] = "IMAGE"; -constexpr char kInputGuideTag[] = "GUIDE"; -constexpr char kOutputFrameTag[] = "IMAGE"; - -constexpr char kInputFrameTagGpu[] = "IMAGE_GPU"; -constexpr char kInputGuideTagGpu[] = "GUIDE_GPU"; -constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU"; - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -} // namespace - -// A calculator for applying a bilateral filter to an image, -// with an optional guide image (joint blateral). -// -// Inputs: -// One of the following two IMAGE tags: -// IMAGE: ImageFrame containing input image - Grayscale or RGB only. -// IMAGE_GPU: GpuBuffer containing input image - Grayscale, RGB or RGBA. -// -// GUIDE (optional): ImageFrame guide image used to filter IMAGE. (N/A). -// GUIDE_GPU (optional): GpuBuffer guide image used to filter IMAGE_GPU. -// -// Output: -// One of the following two tags: -// IMAGE: A filtered ImageFrame - Same as input. -// IMAGE_GPU: A filtered GpuBuffer - RGBA -// -// Options: -// sigma_space: Pixel radius: use (sigma_space*2+1)x(sigma_space*2+1) window. -// This should be set based on output image pixel space. -// sigma_color: Color variance: normalized [0-1] color difference allowed. -// -// Notes: -// * When GUIDE is present, the output image is same size as GUIDE image; -// otherwise, the output image is same size as input image. -// * On GPU the kernel window is subsampled by approximately sqrt(sigma_space) -// i.e. the step size is ~sqrt(sigma_space), -// prioritizing performance > quality. -// * TODO: Add CPU path for joint filter. -// -class BilateralFilterCalculator : public CalculatorBase { - public: - BilateralFilterCalculator() = default; - ~BilateralFilterCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - - absl::Status GlSetup(CalculatorContext* cc); - void GlRender(CalculatorContext* cc); - - mediapipe::BilateralFilterCalculatorOptions options_; - float sigma_color_ = -1.f; - float sigma_space_ = -1.f; - - bool use_gpu_ = false; - bool gpu_initialized_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint program_ = 0; - GLuint vao_; - GLuint vbo_[2]; // vertex storage -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(BilateralFilterCalculator); - -absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); - - if (cc->Inputs().HasTag(kInputFrameTag) && - cc->Inputs().HasTag(kInputFrameTagGpu)) { - return absl::InternalError("Cannot have multiple input images."); - } - if (cc->Inputs().HasTag(kInputFrameTagGpu) != - cc->Outputs().HasTag(kOutputFrameTagGpu)) { - return absl::InternalError("GPU output must have GPU input."); - } - - bool use_gpu = false; - - // Input image to filter. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputFrameTagGpu)) { - cc->Inputs().Tag(kInputFrameTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputFrameTag)) { - cc->Inputs().Tag(kInputFrameTag).Set(); - } - - // Input guide image mask (optional) -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputGuideTagGpu)) { - cc->Inputs().Tag(kInputGuideTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputGuideTag)) { - cc->Inputs().Tag(kInputGuideTag).Set(); - } - - // Output image. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kOutputFrameTagGpu)) { - cc->Outputs().Tag(kOutputFrameTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kOutputFrameTag)) { - cc->Outputs().Tag(kOutputFrameTag).Set(); - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - - if (cc->Inputs().HasTag(kInputFrameTagGpu) && - cc->Outputs().HasTag(kOutputFrameTagGpu)) { -#if !MEDIAPIPE_DISABLE_GPU - use_gpu_ = true; -#else - RET_CHECK_FAIL() << "GPU processing not enabled."; -#endif - } - - sigma_color_ = options_.sigma_color(); - sigma_space_ = options_.sigma_space(); - CHECK_GE(sigma_color_, 0.0); - CHECK_GE(sigma_space_, 0.0); - if (!use_gpu_) sigma_color_ *= 255.0; - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status BilateralFilterCalculator::Process(CalculatorContext* cc) { - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(GlSetup(cc)); - gpu_initialized_ = true; - } - MP_RETURN_IF_ERROR(RenderGpu(cc)); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(RenderCpu(cc)); - } - - return absl::OkStatus(); -} - -absl::Status BilateralFilterCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - if (vao_) glDeleteVertexArrays(1, &vao_); - if (vbo_[0]) glDeleteBuffers(2, vbo_); - program_ = 0; - vao_ = 0; - vbo_[0] = 0; - vbo_[1] = 0; - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status BilateralFilterCalculator::RenderCpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kInputFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get(); - auto input_mat = mediapipe::formats::MatView(&input_frame); - - // Only 1 or 3 channel images supported by OpenCV. - if ((input_mat.channels() == 1 || input_mat.channels() == 3)) { - return absl::InternalError( - "CPU filtering supports only 1 or 3 channel input images."); - } - - auto output_frame = absl::make_unique( - input_frame.Format(), input_mat.cols, input_mat.rows); - const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTag) && - !cc->Inputs().Tag(kInputGuideTag).IsEmpty(); - - if (has_guide_image) { - // cv::jointBilateralFilter() is in contrib module 'ximgproc'. - return absl::UnimplementedError( - "CPU joint filtering support is not implemented yet."); - } else { - auto output_mat = mediapipe::formats::MatView(output_frame.get()); - // Prefer setting 'd = sigma_space * 2' to match GPU definition of radius. - cv::bilateralFilter(input_mat, output_mat, /*d=*/sigma_space_ * 2.0, - sigma_color_, sigma_space_); - } - - cc->Outputs() - .Tag(kOutputFrameTag) - .Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -absl::Status BilateralFilterCalculator::RenderGpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) { - return absl::OkStatus(); - } -#if !MEDIAPIPE_DISABLE_GPU - const auto& input_frame = - cc->Inputs().Tag(kInputFrameTagGpu).Get(); - auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); - - mediapipe::GlTexture output_texture; - const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu); - - // Setup textures and Update image in GPU shader. - if (has_guide_image) { - if (cc->Inputs().Tag(kInputGuideTagGpu).IsEmpty()) return absl::OkStatus(); - // joint bilateral filter - glUseProgram(program_); - const auto& guide_image = - cc->Inputs().Tag(kInputGuideTagGpu).Get(); - auto guide_texture = gpu_helper_.CreateSourceTexture(guide_image); - glUniform2f(glGetUniformLocation(program_, "texel_size_guide"), - 1.0 / guide_image.width(), 1.0 / guide_image.height()); - output_texture = gpu_helper_.CreateDestinationTexture( - guide_image.width(), guide_image.height(), - mediapipe::GpuBufferFormat::kBGRA32); - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_texture.name()); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, guide_texture.name()); - GlRender(cc); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - guide_texture.Release(); - } else { - // regular bilateral filter - glUseProgram(program_); - glUniform2f(glGetUniformLocation(program_, "texel_size"), - 1.0 / input_frame.width(), 1.0 / input_frame.height()); - output_texture = gpu_helper_.CreateDestinationTexture( - input_frame.width(), input_frame.height(), - mediapipe::GpuBufferFormat::kBGRA32); - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_texture.name()); - GlRender(cc); - glBindTexture(GL_TEXTURE_2D, 0); - } - glFlush(); - - // Send out image as GPU packet. - auto output_frame = output_texture.GetFrame(); - cc->Outputs() - .Tag(kOutputFrameTagGpu) - .Add(output_frame.release(), cc->InputTimestamp()); - - // Cleanup - input_texture.Release(); - output_texture.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void BilateralFilterCalculator::GlRender(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - // bring back vao and vbo - glBindVertexArray(vao_); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glBindVertexArray(0); -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status BilateralFilterCalculator::GlSetup(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Common functions and settings for both shaders. - const std::string common_string = - absl::StrReplaceAll(R"( - const float sigma_space = $space; - const float sigma_color = $color; - - const float kSparsityFactor = 0.66; // Higher is more sparse. - const float sparsity = max(1.0, sqrt(sigma_space) * kSparsityFactor); - const float step = sparsity; - const float radius = sigma_space; - const float offset = (step > 1.0) ? (step * 0.5) : (0.0); - - float gaussian(float x, float sigma) { - float coeff = -0.5 / (sigma * sigma * 4.0 + 1.0e-6); - return exp((x * x) * coeff); - } - )", - {{"$space", std::to_string(sigma_space_)}, - {"$color", std::to_string(sigma_color_)}}); - - // Shader to do bilateral filtering on input image based on sigma space/color. - // Large kernel sizes are subsampled based on sqrt(sigma_space) window size, - // denoted as 'sparsity' below. - const std::string frag_src = - std::string(mediapipe::kMediaPipeFragmentShaderPreamble) + R"( - DEFAULT_PRECISION(highp, float) - - in vec2 sample_coordinate; - uniform sampler2D input_frame; - uniform vec2 texel_size; - - )" + - common_string + R"( - - void main() { - vec2 center_uv = sample_coordinate; - vec3 center_val = texture2D(input_frame, center_uv).rgb; - vec3 new_val = vec3(0.0); - - float space_weight = 0.0; - float color_weight = 0.0; - float total_weight = 0.0; - - float sigma_texel = max(texel_size.x, texel_size.y) * sigma_space; - // Subsample kernel space. - for (float i = -radius+offset; i <= radius; i+=step) { - for (float j = -radius+offset; j <= radius; j+=step) { - vec2 shift = vec2(j, i) * texel_size; - vec2 uv = vec2(center_uv + shift); - vec3 val = texture2D(input_frame, uv).rgb; - - space_weight = gaussian(distance(center_uv, uv), sigma_texel); - color_weight = gaussian(distance(center_val, val), sigma_color); - total_weight += space_weight * color_weight; - - new_val += vec3(space_weight * color_weight) * val; - } - } - new_val /= vec3(total_weight); - - gl_FragColor = vec4(new_val, 1.0); - } - )"; - - // Shader to do joint bilateral filtering on input image based on - // sigma space/color, and a Guide image. - // Large kernel sizes are subsampled based on sqrt(sigma_space) window size, - // denoted as 'sparsity' below. - const std::string joint_frag_src = - std::string(mediapipe::kMediaPipeFragmentShaderPreamble) + R"( - DEFAULT_PRECISION(highp, float) - - in vec2 sample_coordinate; - uniform sampler2D input_frame; - uniform sampler2D guide_frame; - uniform vec2 texel_size_guide; // size of guide and resulting filtered image - - )" + - common_string + R"( - - void main() { - vec2 center_uv = sample_coordinate; - vec3 center_val = texture2D(guide_frame, center_uv).rgb; - vec3 new_val = vec3(0.0); - - float space_weight = 0.0; - float color_weight = 0.0; - float total_weight = 0.0; - - float sigma_texel = max(texel_size_guide.x, texel_size_guide.y) * sigma_space; - // Subsample kernel space. - for (float i = -radius+offset; i <= radius; i+=step) { - for (float j = -radius+offset; j <= radius; j+=step) { - vec2 shift = vec2(j, i) * texel_size_guide; - vec2 uv = vec2(center_uv + shift); - vec3 guide_val = texture2D(guide_frame, uv).rgb; - vec3 out_val = texture2D(input_frame, uv).rgb; - - space_weight = gaussian(distance(center_uv, uv), sigma_texel); - color_weight = gaussian(distance(center_val, guide_val), sigma_color); - total_weight += space_weight * color_weight; - - new_val += vec3(space_weight * color_weight) * out_val; - } - } - new_val /= vec3(total_weight); - - gl_FragColor = vec4(new_val, 1.0); - } - )"; - - // Only initialize the one shader to be used. - const bool has_guide_image = cc->Inputs().HasTag(kInputGuideTagGpu); - - if (has_guide_image) { - // Create joint shader program and set parameters. - mediapipe::GlhCreateProgram( - mediapipe::kBasicVertexShader, joint_frag_src.c_str(), NUM_ATTRIBUTES, - (const GLchar**)&attr_name[0], attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); - glUniform1i(glGetUniformLocation(program_, "guide_frame"), 2); - } else { - // Create default shader program and set parameters. - mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(), - NUM_ATTRIBUTES, (const GLchar**)&attr_name[0], - attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); - } - - // Generate vbos and vao. - glGenVertexArrays(1, &vao_); - glGenBuffers(2, vbo_); - - // Fill in static vbo (vbo 0), to be reused in GlRender(). - glBindVertexArray(vao_); - glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), - mediapipe::kBasicSquareVertices, GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - glBindBuffer(GL_ARRAY_BUFFER, 0); - // Fill in static vbo (vbo 1), to be reused in GlRender(). - glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), - mediapipe::kBasicTextureVertices, GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/bilateral_filter_calculator.proto b/mediapipe/calculators/image/bilateral_filter_calculator.proto deleted file mode 100644 index a787437dc..000000000 --- a/mediapipe/calculators/image/bilateral_filter_calculator.proto +++ /dev/null @@ -1,20 +0,0 @@ -// Options for BilateralFilterCalculator -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message BilateralFilterCalculatorOptions { - extend CalculatorOptions { - optional BilateralFilterCalculatorOptions ext = 255670209; - } - - // Max variance in color allowed, based on normalized color values. - optional float sigma_color = 1; - - // Window radius. - // Results in a '(sigma_space*2+1) x (sigma_space*2+1)' size kernel. - // This should be set based on output image pixel space. - optional float sigma_space = 2; -} diff --git a/mediapipe/calculators/image/color_convert_calculator.cc b/mediapipe/calculators/image/color_convert_calculator.cc deleted file mode 100644 index bdac932bb..000000000 --- a/mediapipe/calculators/image/color_convert_calculator.cc +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/source_location.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { -namespace { -void SetColorChannel(int channel, uint8 value, cv::Mat* mat) { - CHECK(mat->depth() == CV_8U); - CHECK(channel < mat->channels()); - const int step = mat->channels(); - for (int r = 0; r < mat->rows; ++r) { - uint8* row_ptr = mat->ptr(r); - for (int offset = channel; offset < mat->cols * step; offset += step) { - row_ptr[offset] = value; - } - } -} - -constexpr char kRgbaInTag[] = "RGBA_IN"; -constexpr char kRgbInTag[] = "RGB_IN"; -constexpr char kBgraInTag[] = "BGRA_IN"; -constexpr char kGrayInTag[] = "GRAY_IN"; -constexpr char kRgbaOutTag[] = "RGBA_OUT"; -constexpr char kRgbOutTag[] = "RGB_OUT"; -constexpr char kBgraOutTag[] = "BGRA_OUT"; -constexpr char kGrayOutTag[] = "GRAY_OUT"; -} // namespace - -// A portable color conversion calculator calculator. -// -// The following conversions are currently supported, but it's fairly easy to -// add new ones if this doesn't meet your needs--Don't forget to add a test to -// color_convert_calculator_test.cc if you do! -// RGBA -> RGB -// GRAY -> RGB -// RGB -> GRAY -// RGB -> RGBA -// RGBA -> BGRA -// BGRA -> RGBA -// -// This calculator only supports a single input stream and output stream at a -// time. If more than one input stream or output stream is present, the -// calculator will fail at FillExpectations. -// TODO: Remove this requirement by replacing the typed input streams -// with a single generic input and allow multiple simultaneous outputs. -// -// Input streams: -// RGBA_IN: The input video stream (ImageFrame, SRGBA). -// RGB_IN: The input video stream (ImageFrame, SRGB). -// BGRA_IN: The input video stream (ImageFrame, SBGRA). -// GRAY_IN: The input video stream (ImageFrame, GRAY8). -// -// Output streams: -// RGBA_OUT: The output video stream (ImageFrame, SRGBA). -// RGB_OUT: The output video stream (ImageFrame, SRGB). -// BGRA_OUT: The output video stream (ImageFrame, SBGRA). -// GRAY_OUT: The output video stream (ImageFrame, GRAY8). -class ColorConvertCalculator : public CalculatorBase { - public: - ~ColorConvertCalculator() override = default; - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Process(CalculatorContext* cc) override; - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - private: - // Wrangles the appropriate inputs and outputs to perform the color - // conversion. The ImageFrame on input_tag is converted using the - // open_cv_convert_code provided and then output on the output_tag stream. - // Note that the output_format must match the destination conversion code. - absl::Status ConvertAndOutput(const std::string& input_tag, - const std::string& output_tag, - ImageFormat::Format output_format, - int open_cv_convert_code, - CalculatorContext* cc); -}; - -REGISTER_CALCULATOR(ColorConvertCalculator); - -absl::Status ColorConvertCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is allowed."; - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is allowed."; - - if (cc->Inputs().HasTag(kRgbaInTag)) { - cc->Inputs().Tag(kRgbaInTag).Set(); - } - - if (cc->Inputs().HasTag(kGrayInTag)) { - cc->Inputs().Tag(kGrayInTag).Set(); - } - - if (cc->Inputs().HasTag(kRgbInTag)) { - cc->Inputs().Tag(kRgbInTag).Set(); - } - - if (cc->Inputs().HasTag(kBgraInTag)) { - cc->Inputs().Tag(kBgraInTag).Set(); - } - - if (cc->Outputs().HasTag(kRgbOutTag)) { - cc->Outputs().Tag(kRgbOutTag).Set(); - } - - if (cc->Outputs().HasTag(kGrayOutTag)) { - cc->Outputs().Tag(kGrayOutTag).Set(); - } - - if (cc->Outputs().HasTag(kRgbaOutTag)) { - cc->Outputs().Tag(kRgbaOutTag).Set(); - } - - if (cc->Outputs().HasTag(kBgraOutTag)) { - cc->Outputs().Tag(kBgraOutTag).Set(); - } - - return absl::OkStatus(); -} - -absl::Status ColorConvertCalculator::ConvertAndOutput( - const std::string& input_tag, const std::string& output_tag, - ImageFormat::Format output_format, int open_cv_convert_code, - CalculatorContext* cc) { - const cv::Mat& input_mat = - formats::MatView(&cc->Inputs().Tag(input_tag).Get()); - std::unique_ptr output_frame( - new ImageFrame(output_format, input_mat.cols, input_mat.rows)); - cv::Mat output_mat = formats::MatView(output_frame.get()); - cv::cvtColor(input_mat, output_mat, open_cv_convert_code); - - // cv::cvtColor will leave the alpha channel set to 0, which is a bizarre - // design choice. Instead, let's set alpha to 255. - if (open_cv_convert_code == cv::COLOR_RGB2RGBA) { - SetColorChannel(3, 255, &output_mat); - } - cc->Outputs() - .Tag(output_tag) - .Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -absl::Status ColorConvertCalculator::Process(CalculatorContext* cc) { - // RGBA -> RGB - if (cc->Inputs().HasTag(kRgbaInTag) && cc->Outputs().HasTag(kRgbOutTag)) { - return ConvertAndOutput(kRgbaInTag, kRgbOutTag, ImageFormat::SRGB, - cv::COLOR_RGBA2RGB, cc); - } - // GRAY -> RGB - if (cc->Inputs().HasTag(kGrayInTag) && cc->Outputs().HasTag(kRgbOutTag)) { - return ConvertAndOutput(kGrayInTag, kRgbOutTag, ImageFormat::SRGB, - cv::COLOR_GRAY2RGB, cc); - } - // RGB -> GRAY - if (cc->Inputs().HasTag(kRgbInTag) && cc->Outputs().HasTag(kGrayOutTag)) { - return ConvertAndOutput(kRgbInTag, kGrayOutTag, ImageFormat::GRAY8, - cv::COLOR_RGB2GRAY, cc); - } - // RGB -> RGBA - if (cc->Inputs().HasTag(kRgbInTag) && cc->Outputs().HasTag(kRgbaOutTag)) { - return ConvertAndOutput(kRgbInTag, kRgbaOutTag, ImageFormat::SRGBA, - cv::COLOR_RGB2RGBA, cc); - } - // BGRA -> RGBA - if (cc->Inputs().HasTag(kBgraInTag) && cc->Outputs().HasTag(kRgbaOutTag)) { - return ConvertAndOutput(kBgraInTag, kRgbaOutTag, ImageFormat::SRGBA, - cv::COLOR_BGRA2RGBA, cc); - } - // RGBA -> BGRA - if (cc->Inputs().HasTag(kRgbaInTag) && cc->Outputs().HasTag(kBgraOutTag)) { - return ConvertAndOutput(kRgbaInTag, kBgraOutTag, ImageFormat::SBGRA, - cv::COLOR_RGBA2BGRA, cc); - } - - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported image format conversion."; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/feature_detector_calculator.cc b/mediapipe/calculators/image/feature_detector_calculator.cc deleted file mode 100644 index 389a33696..000000000 --- a/mediapipe/calculators/image/feature_detector_calculator.cc +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/synchronization/blocking_counter.h" -#include "mediapipe/calculators/image/feature_detector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_features2d_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/threadpool.h" -#include "mediapipe/framework/tool/options_util.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { - -const char kOptionsTag[] = "OPTIONS"; -const int kPatchSize = 32; -const int kNumThreads = 16; - -// A calculator to apply local feature detection. -// Input stream: -// IMAGE: Input image frame of type ImageFrame from video stream. -// Output streams: -// FEATURES: The detected keypoints from input image as vector. -// PATCHES: Optional output the extracted patches as vector -class FeatureDetectorCalculator : public CalculatorBase { - public: - ~FeatureDetectorCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - FeatureDetectorCalculatorOptions options_; - cv::Ptr feature_detector_; - std::unique_ptr pool_; - - // Create image pyramid based on input image. - void ComputeImagePyramid(const cv::Mat& input_image, - std::vector* image_pyramid); - - // Extract the patch for single feature with image pyramid. - cv::Mat ExtractPatch(const cv::KeyPoint& feature, - const std::vector& image_pyramid); -}; - -REGISTER_CALCULATOR(FeatureDetectorCalculator); - -absl::Status FeatureDetectorCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag("IMAGE")) { - cc->Inputs().Tag("IMAGE").Set(); - } - if (cc->Outputs().HasTag("FEATURES")) { - cc->Outputs().Tag("FEATURES").Set>(); - } - if (cc->Outputs().HasTag("LANDMARKS")) { - cc->Outputs().Tag("LANDMARKS").Set(); - } - if (cc->Outputs().HasTag("PATCHES")) { - cc->Outputs().Tag("PATCHES").Set>(); - } - return absl::OkStatus(); -} - -absl::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) { - options_ = - tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag) - .GetExtension(FeatureDetectorCalculatorOptions::ext); - feature_detector_ = cv::ORB::create( - options_.max_features(), options_.scale_factor(), - options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE); - pool_ = absl::make_unique("ThreadPool", kNumThreads); - pool_->StartWorkers(); - return absl::OkStatus(); -} - -absl::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) { - const Timestamp& timestamp = cc->InputTimestamp(); - if (timestamp == Timestamp::PreStream()) { - // Indicator packet. - return absl::OkStatus(); - } - InputStream* input_frame = &(cc->Inputs().Tag("IMAGE")); - cv::Mat input_view = formats::MatView(&input_frame->Get()); - cv::Mat grayscale_view; - cv::cvtColor(input_view, grayscale_view, cv::COLOR_RGB2GRAY); - - std::vector keypoints; - feature_detector_->detect(grayscale_view, keypoints); - if (keypoints.size() > options_.max_features()) { - keypoints.resize(options_.max_features()); - } - - if (cc->Outputs().HasTag("FEATURES")) { - auto features_ptr = absl::make_unique>(keypoints); - cc->Outputs().Tag("FEATURES").Add(features_ptr.release(), timestamp); - } - - if (cc->Outputs().HasTag("LANDMARKS")) { - auto landmarks_ptr = absl::make_unique(); - for (int j = 0; j < keypoints.size(); ++j) { - auto feature_landmark = landmarks_ptr->add_landmark(); - feature_landmark->set_x(keypoints[j].pt.x / grayscale_view.cols); - feature_landmark->set_y(keypoints[j].pt.y / grayscale_view.rows); - } - cc->Outputs().Tag("LANDMARKS").Add(landmarks_ptr.release(), timestamp); - } - - if (cc->Outputs().HasTag("PATCHES")) { - std::vector image_pyramid; - ComputeImagePyramid(grayscale_view, &image_pyramid); - std::vector patch_mat; - patch_mat.resize(keypoints.size()); - absl::BlockingCounter counter(keypoints.size()); - for (int i = 0; i < keypoints.size(); i++) { - pool_->Schedule( - [this, &image_pyramid, &keypoints, &patch_mat, i, &counter] { - patch_mat[i] = ExtractPatch(keypoints[i], image_pyramid); - counter.DecrementCount(); - }); - } - counter.Wait(); - const int batch_size = options_.max_features(); - auto patches = absl::make_unique>(); - TfLiteTensor tensor; - tensor.type = kTfLiteFloat32; - tensor.dims = TfLiteIntArrayCreate(4); - tensor.dims->data[0] = batch_size; - tensor.dims->data[1] = kPatchSize; - tensor.dims->data[2] = kPatchSize; - tensor.dims->data[3] = 1; - int num_bytes = batch_size * kPatchSize * kPatchSize * sizeof(float); - tensor.data.data = malloc(num_bytes); - tensor.bytes = num_bytes; - tensor.allocation_type = kTfLiteArenaRw; - float* tensor_buffer = tensor.data.f; - for (int i = 0; i < keypoints.size(); i++) { - for (int j = 0; j < patch_mat[i].rows; ++j) { - for (int k = 0; k < patch_mat[i].cols; ++k) { - *tensor_buffer++ = patch_mat[i].at(j, k) / 128.0f - 1.0f; - } - } - } - for (int i = keypoints.size() * kPatchSize * kPatchSize; i < num_bytes / 4; - i++) { - *tensor_buffer++ = 0; - } - - patches->emplace_back(tensor); - cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp); - } - - return absl::OkStatus(); -} - -void FeatureDetectorCalculator::ComputeImagePyramid( - const cv::Mat& input_image, std::vector* image_pyramid) { - cv::Mat tmp_image = input_image; - cv::Mat src_image = input_image; - for (int i = 0; i < options_.pyramid_level(); ++i) { - image_pyramid->push_back(src_image); - cv::resize(src_image, tmp_image, cv::Size(), 1.0f / options_.scale_factor(), - 1.0f / options_.scale_factor()); - src_image = tmp_image; - } -} - -cv::Mat FeatureDetectorCalculator::ExtractPatch( - const cv::KeyPoint& feature, const std::vector& image_pyramid) { - cv::Mat img = image_pyramid[feature.octave]; - float scale_factor = 1 / pow(options_.scale_factor(), feature.octave); - cv::Point2f center = - cv::Point2f(feature.pt.x * scale_factor, feature.pt.y * scale_factor); - cv::Mat rot = cv::getRotationMatrix2D(center, feature.angle, 1.0); - rot.at(0, 2) += kPatchSize / 2 - center.x; - rot.at(1, 2) += kPatchSize / 2 - center.y; - cv::Mat cropped_img; - // perform the affine transformation - cv::warpAffine(img, cropped_img, rot, cv::Size(kPatchSize, kPatchSize), - cv::INTER_LINEAR); - return cropped_img; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/feature_detector_calculator.proto b/mediapipe/calculators/image/feature_detector_calculator.proto deleted file mode 100644 index 7bff23572..000000000 --- a/mediapipe/calculators/image/feature_detector_calculator.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Options for FeatureDetectorCalculator -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message FeatureDetectorCalculatorOptions { - extend CalculatorOptions { - optional FeatureDetectorCalculatorOptions ext = 278741680; - } - - // Set to true if output patches, otherwise only output cv::KeyPoint - optional bool output_patch = 1; - - // The max number of detected features. - optional int32 max_features = 2 [default = 200]; - - // The number of pyramid levels. - optional int32 pyramid_level = 3 [default = 4]; - - // Pyramid decimation ratio. - optional float scale_factor = 4 [default = 1.2]; -} diff --git a/mediapipe/calculators/image/image_clone_calculator.cc b/mediapipe/calculators/image/image_clone_calculator.cc deleted file mode 100644 index 107c42b92..000000000 --- a/mediapipe/calculators/image/image_clone_calculator.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/image_clone_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/port/status.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { -namespace api2 { - -#if MEDIAPIPE_DISABLE_GPU -// Just a placeholder to not have to depend on mediapipe::GpuBuffer. -using GpuBuffer = AnyType; -#else -using GpuBuffer = mediapipe::GpuBuffer; -#endif // MEDIAPIPE_DISABLE_GPU - -// Clones an input image and makes sure in the output clone the pixel data are -// stored on the target storage (CPU vs GPU) specified in the calculator option. -// -// The clone shares ownership of the input pixel data on the existing storage. -// If the target storage is diffrent from the existing one, then the data is -// further copied there. -// -// Example usage: -// node { -// calculator: "ImageCloneCalculator" -// input_stream: "input" -// output_stream: "output" -// options: { -// [mediapipe.ImageCloneCalculatorOptions.ext] { -// output_on_gpu: true -// } -// } -// } -class ImageCloneCalculator : public Node { - public: - static constexpr Input kIn{""}; - static constexpr Output kOut{""}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { -#if MEDIAPIPE_DISABLE_GPU - if (cc->Options().output_on_gpu()) { - return absl::UnimplementedError( - "GPU processing is disabled in build flags"); - } -#else - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // MEDIAPIPE_DISABLE_GPU - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const auto& options = cc->Options(); - output_on_gpu_ = options.output_on_gpu(); -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - std::unique_ptr output; - const auto& input = *kIn(cc); - if (input.UsesGpu()) { -#if !MEDIAPIPE_DISABLE_GPU - // Create an output Image that co-owns the underlying texture buffer as - // the input Image. - output = std::make_unique(input.GetGpuBuffer()); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - // Make a copy of the input packet to co-own the input Image. - mediapipe::Packet* packet_copy_ptr = - new mediapipe::Packet(kIn(cc).packet()); - // Create an output Image that (co-)owns a new ImageFrame that points to - // the same pixel data as the input Image and also owns the packet - // copy. As a result, the output Image indirectly co-owns the input - // Image. This ensures a correct life span of the shared pixel data. - output = std::make_unique(std::make_unique( - input.image_format(), input.width(), input.height(), input.step(), - const_cast(input.GetImageFrameSharedPtr()->PixelData()), - [packet_copy_ptr](uint8*) { delete packet_copy_ptr; })); - } - - if (output_on_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); }); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - output->ConvertToCpu(); - } - kOut(cc).Send(std::move(output)); - - return absl::OkStatus(); - } - - private: - bool output_on_gpu_; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -MEDIAPIPE_REGISTER_NODE(ImageCloneCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_clone_calculator.proto b/mediapipe/calculators/image/image_clone_calculator.proto deleted file mode 100644 index 6fa05dccf..000000000 --- a/mediapipe/calculators/image/image_clone_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message ImageCloneCalculatorOptions { - extend CalculatorOptions { - optional ImageCloneCalculatorOptions ext = 372781894; - } - - // Whether the output clone should have pixel data already available on GPU. - optional bool output_on_gpu = 1 [default = false]; -} diff --git a/mediapipe/calculators/image/image_cropping_calculator.cc b/mediapipe/calculators/image/image_cropping_calculator.cc deleted file mode 100644 index 07f7d5f46..000000000 --- a/mediapipe/calculators/image/image_cropping_calculator.cc +++ /dev/null @@ -1,555 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/image_cropping_calculator.h" - -#include - -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace { -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -} // namespace - -namespace mediapipe { - -namespace { - -#if !MEDIAPIPE_DISABLE_GPU - -#endif // !MEDIAPIPE_DISABLE_GPU - -constexpr char kRectTag[] = "RECT"; -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kHeightTag[] = "HEIGHT"; -constexpr char kImageTag[] = "IMAGE"; -constexpr char kImageGpuTag[] = "IMAGE_GPU"; -constexpr char kWidthTag[] = "WIDTH"; - -} // namespace - -REGISTER_CALCULATOR(ImageCroppingCalculator); - -absl::Status ImageCroppingCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kImageTag) ^ cc->Inputs().HasTag(kImageGpuTag)); - RET_CHECK(cc->Outputs().HasTag(kImageTag) ^ - cc->Outputs().HasTag(kImageGpuTag)); - - bool use_gpu = false; - - if (cc->Inputs().HasTag(kImageTag)) { - RET_CHECK(cc->Outputs().HasTag(kImageTag)); - cc->Inputs().Tag(kImageTag).Set(); - cc->Outputs().Tag(kImageTag).Set(); - } -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kImageGpuTag)) { - RET_CHECK(cc->Outputs().HasTag(kImageGpuTag)); - cc->Inputs().Tag(kImageGpuTag).Set(); - cc->Outputs().Tag(kImageGpuTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - - int flags = 0; - if (cc->Inputs().HasTag(kRectTag)) { - ++flags; - } - if (cc->Inputs().HasTag(kWidthTag) && cc->Inputs().HasTag(kHeightTag)) { - ++flags; - } - if (cc->Inputs().HasTag(kNormRectTag)) { - ++flags; - } - if (cc->Options() - .has_norm_width() && - cc->Options() - .has_norm_height()) { - ++flags; - } - if (cc->Options().has_width() && - cc->Options().has_height()) { - ++flags; - } - RET_CHECK(flags == 1) << "Illegal combination of input streams/options."; - - if (cc->Inputs().HasTag(kRectTag)) { - cc->Inputs().Tag(kRectTag).Set(); - } - if (cc->Inputs().HasTag(kNormRectTag)) { - cc->Inputs().Tag(kNormRectTag).Set(); - } - if (cc->Inputs().HasTag(kWidthTag)) { - cc->Inputs().Tag(kWidthTag).Set(); - } - if (cc->Inputs().HasTag(kHeightTag)) { - cc->Inputs().Tag(kHeightTag).Set(); - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kImageGpuTag)) { - use_gpu_ = true; - } - - options_ = cc->Options(); - output_max_width_ = - options_.has_output_max_width() ? options_.output_max_width() : FLT_MAX; - output_max_height_ = - options_.has_output_max_height() ? options_.output_max_height() : FLT_MAX; - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#else - RET_CHECK_FAIL() << "GPU processing is for Android and iOS only."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - // Validate border mode. - if (use_gpu_) { - MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc)); - } else { - MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc)); - } - - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kRectTag) && cc->Inputs().Tag(kRectTag).IsEmpty()) { - VLOG(1) << "RECT is empty for timestamp: " << cc->InputTimestamp(); - return absl::OkStatus(); - } - if (cc->Inputs().HasTag(kNormRectTag) && - cc->Inputs().Tag(kNormRectTag).IsEmpty()) { - VLOG(1) << "NORM_RECT is empty for timestamp: " << cc->InputTimestamp(); - return absl::OkStatus(); - } - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(InitGpu(cc)); - gpu_initialized_ = true; - } - MP_RETURN_IF_ERROR(RenderGpu(cc)); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(RenderCpu(cc)); - } - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - program_ = 0; - }); - gpu_initialized_ = false; -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::ValidateBorderModeForCPU( - CalculatorContext* cc) { - int border_mode; - return GetBorderModeForOpenCV(cc, &border_mode); -} - -absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU( - CalculatorContext* cc) { - mediapipe::ImageCroppingCalculatorOptions options = - cc->Options(); - - switch (options.border_mode()) { - case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: - LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU " - << "implementation and will fall back into BORDER_REPLICATE"; - break; - case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: - break; - default: - RET_CHECK_FAIL() << "Unsupported border mode for GPU: " - << options.border_mode(); - } - - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kImageTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto& input_img = cc->Inputs().Tag(kImageTag).Get(); - cv::Mat input_mat = formats::MatView(&input_img); - - RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height()); - int target_width = specs.width, target_height = specs.height, - rect_center_x = specs.center_x, rect_center_y = specs.center_y; - float rotation = specs.rotation; - - // Get border mode and value for OpenCV. - int border_mode; - MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode)); - - const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y), - cv::Size2f(target_width, target_height), - rotation * 180.f / M_PI); - cv::Mat src_points; - cv::boxPoints(min_rect, src_points); - - float output_width = min_rect.size.width; - float output_height = min_rect.size.height; - float scale = std::min({1.0f, output_max_width_ / output_width, - output_max_height_ / output_height}); - output_width *= scale; - output_height *= scale; - - float dst_corners[8] = {0, - output_height - 1, - 0, - 0, - output_width - 1, - 0, - output_width - 1, - output_height - 1}; - cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners); - cv::Mat projection_matrix = - cv::getPerspectiveTransform(src_points, dst_points); - cv::Mat cropped_image; - cv::warpPerspective(input_mat, cropped_image, projection_matrix, - cv::Size(output_width, output_height), - /* flags = */ 0, - /* borderMode = */ border_mode); - - std::unique_ptr output_frame(new ImageFrame( - input_img.Format(), cropped_image.cols, cropped_image.rows)); - cv::Mat output_mat = formats::MatView(output_frame.get()); - cropped_image.copyTo(output_mat); - cc->Outputs().Tag(kImageTag).Add(output_frame.release(), - cc->InputTimestamp()); - return absl::OkStatus(); -} - -absl::Status ImageCroppingCalculator::RenderGpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) { - return absl::OkStatus(); - } -#if !MEDIAPIPE_DISABLE_GPU - const Packet& input_packet = cc->Inputs().Tag(kImageGpuTag).Value(); - const auto& input_buffer = input_packet.Get(); - auto src_tex = gpu_helper_.CreateSourceTexture(input_buffer); - - int out_width, out_height; - GetOutputDimensions(cc, src_tex.width(), src_tex.height(), &out_width, - &out_height); - auto dst_tex = gpu_helper_.CreateDestinationTexture(out_width, out_height); - - // Run cropping shader on GPU. - { - gpu_helper_.BindFramebuffer(dst_tex); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(src_tex.target(), src_tex.name()); - - GlRender(); - - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } - - // Send result image in GPU packet. - auto output = dst_tex.GetFrame(); - cc->Outputs().Tag(kImageGpuTag).Add(output.release(), cc->InputTimestamp()); - - // Cleanup - src_tex.Release(); - dst_tex.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void ImageCroppingCalculator::GlRender() { -#if !MEDIAPIPE_DISABLE_GPU - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - const GLfloat* texture_vertices = &transformed_points_[0]; - - // program - glUseProgram(program_); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); - -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status ImageCroppingCalculator::InitGpu(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Simple pass-through shader. - const GLchar* frag_src = GLES_VERSION_COMPAT - R"( - #if __VERSION__ < 130 - #define in varying - #endif // __VERSION__ < 130 - - #ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; - #else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; - #endif // defined(GL_ES) - - in vec2 sample_coordinate; - uniform sampler2D input_frame; - - void main() { - vec4 pix = texture2D(input_frame, sample_coordinate); - fragColor = pix; - } - )"; - - // Program - mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src, - NUM_ATTRIBUTES, &attr_name[0], attr_location, - &program_); - RET_CHECK(program_) << "Problem initializing the program."; - - // Parameters - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -// For GPU only. -void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc, - int src_width, int src_height, - int* dst_width, - int* dst_height) { - RectSpec specs = GetCropSpecs(cc, src_width, src_height); - int crop_width = specs.width, crop_height = specs.height, - x_center = specs.center_x, y_center = specs.center_y; - float rotation = specs.rotation; - - const float half_width = crop_width / 2.0f; - const float half_height = crop_height / 2.0f; - const float corners[] = {-half_width, -half_height, half_width, -half_height, - -half_width, half_height, half_width, half_height}; - - for (int i = 0; i < 4; ++i) { - const float rotated_x = std::cos(rotation) * corners[i * 2] - - std::sin(rotation) * corners[i * 2 + 1]; - const float rotated_y = std::sin(rotation) * corners[i * 2] + - std::cos(rotation) * corners[i * 2 + 1]; - - transformed_points_[i * 2] = ((rotated_x + x_center) / src_width); - transformed_points_[i * 2 + 1] = ((rotated_y + y_center) / src_height); - } - - // Find the boundaries of the transformed rectangle. - float col_min = transformed_points_[0]; - float col_max = transformed_points_[0]; - float row_min = transformed_points_[1]; - float row_max = transformed_points_[1]; - for (int i = 1; i < 4; ++i) { - col_min = std::min(col_min, transformed_points_[i * 2]); - col_max = std::max(col_max, transformed_points_[i * 2]); - row_min = std::min(row_min, transformed_points_[i * 2 + 1]); - row_max = std::max(row_max, transformed_points_[i * 2 + 1]); - } - - int width = static_cast(std::round((col_max - col_min) * src_width)); - int height = static_cast(std::round((row_max - row_min) * src_height)); - - float scale = - std::min({1.0f, output_max_width_ / width, output_max_height_ / height}); - width *= scale; - height *= scale; - - // Minimum output dimension 1x1 prevents creation of textures with 0x0. - *dst_width = std::max(1, width); - *dst_height = std::max(1, height); -} - -RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc, - int src_width, int src_height) { - // Get the size of the cropping box. - int crop_width = src_width; - int crop_height = src_height; - // Get the center of cropping box. Default is the at the center. - int x_center = src_width / 2; - int y_center = src_height / 2; - // Get the rotation of the cropping box. - float rotation = 0.0f; - // Get the normalized width and height if specified by the inputs or options. - float normalized_width = 0.0f; - float normalized_height = 0.0f; - - mediapipe::ImageCroppingCalculatorOptions options = - cc->Options(); - - // width/height, norm_width/norm_height from input streams take precednece. - if (cc->Inputs().HasTag(kRectTag)) { - const auto& rect = cc->Inputs().Tag(kRectTag).Get(); - // Only use the rect if it is valid. - if (rect.width() > 0 && rect.height() > 0 && rect.x_center() >= 0 && - rect.y_center() >= 0) { - x_center = rect.x_center(); - y_center = rect.y_center(); - crop_width = rect.width(); - crop_height = rect.height(); - rotation = rect.rotation(); - } - } else if (cc->Inputs().HasTag(kNormRectTag)) { - const auto& norm_rect = - cc->Inputs().Tag(kNormRectTag).Get(); - if (norm_rect.width() > 0.0 && norm_rect.height() > 0.0) { - normalized_width = norm_rect.width(); - normalized_height = norm_rect.height(); - x_center = std::round(norm_rect.x_center() * src_width); - y_center = std::round(norm_rect.y_center() * src_height); - rotation = norm_rect.rotation(); - } - } else if (cc->Inputs().HasTag(kWidthTag) && - cc->Inputs().HasTag(kHeightTag)) { - crop_width = cc->Inputs().Tag(kWidthTag).Get(); - crop_height = cc->Inputs().Tag(kHeightTag).Get(); - } else if (options.has_width() && options.has_height()) { - crop_width = options.width(); - crop_height = options.height(); - } else if (options.has_norm_width() && options.has_norm_height()) { - normalized_width = options.norm_width(); - normalized_height = options.norm_height(); - } - - // Get the crop width and height from the normalized width and height. - if (normalized_width > 0 && normalized_height > 0) { - crop_width = std::round(normalized_width * src_width); - crop_height = std::round(normalized_height * src_height); - } - - // Rotation and center values from input streams take precedence, so only - // look at those values in the options if kRectTag and kNormRectTag are not - // present from the inputs. - if (!cc->Inputs().HasTag(kRectTag) && !cc->Inputs().HasTag(kNormRectTag)) { - if (options.has_norm_center_x() && options.has_norm_center_y()) { - x_center = std::round(options.norm_center_x() * src_width); - y_center = std::round(options.norm_center_y() * src_height); - } - if (options.has_rotation()) { - rotation = options.rotation(); - } - } - - return {crop_width, crop_height, x_center, y_center, rotation}; -} - -absl::Status ImageCroppingCalculator::GetBorderModeForOpenCV( - CalculatorContext* cc, int* border_mode) { - mediapipe::ImageCroppingCalculatorOptions options = - cc->Options(); - - switch (options.border_mode()) { - case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: - *border_mode = cv::BORDER_CONSTANT; - break; - case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: - *border_mode = cv::BORDER_REPLICATE; - break; - default: - RET_CHECK_FAIL() << "Unsupported border mode for CPU: " - << options.border_mode(); - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_cropping_calculator.h b/mediapipe/calculators/image/image_cropping_calculator.h deleted file mode 100644 index 39d99cc55..000000000 --- a/mediapipe/calculators/image/image_cropping_calculator.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_ - -#include - -#include "mediapipe/calculators/image/image_cropping_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -// Crops the input texture to the given rectangle region. The rectangle can -// be at arbitrary location on the image with rotation. If there's rotation, the -// output texture will have the size of the input rectangle. The rotation should -// be in radian, see rect.proto for detail. -// -// Input: -// One of the following two tags: -// IMAGE - ImageFrame representing the input image. -// IMAGE_GPU - GpuBuffer representing the input image. -// One of the following two tags (optional if WIDTH/HEIGHT is specified): -// RECT - A Rect proto specifying the width/height and location of the -// cropping rectangle. -// NORM_RECT - A NormalizedRect proto specifying the width/height and location -// of the cropping rectangle in normalized coordinates. -// Alternative tags to RECT (optional if RECT/NORM_RECT is specified): -// WIDTH - The desired width of the output cropped image, -// based on image center -// HEIGHT - The desired height of the output cropped image, -// based on image center -// -// Output: -// One of the following two tags: -// IMAGE - Cropped ImageFrame -// IMAGE_GPU - Cropped GpuBuffer. -// -// Note: input_stream values take precedence over options defined in the graph. -// -namespace mediapipe { - -struct RectSpec { - int width; - int height; - int center_x; - int center_y; - float rotation; - - bool operator==(const RectSpec& rect) const { - return (width == rect.width && height == rect.height && - center_x == rect.center_x && center_y == rect.center_y && - rotation == rect.rotation); - } -}; - -class ImageCroppingCalculator : public CalculatorBase { - public: - ImageCroppingCalculator() = default; - ~ImageCroppingCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - static RectSpec GetCropSpecs(const CalculatorContext* cc, int src_width, - int src_height); - - private: - absl::Status ValidateBorderModeForCPU(CalculatorContext* cc); - absl::Status ValidateBorderModeForGPU(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status InitGpu(CalculatorContext* cc); - void GlRender(); - void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height, - int* dst_width, int* dst_height); - absl::Status GetBorderModeForOpenCV(CalculatorContext* cc, int* border_mode); - - mediapipe::ImageCroppingCalculatorOptions options_; - - bool use_gpu_ = false; - // Output texture corners (4) after transoformation in normalized coordinates. - float transformed_points_[8]; - float output_max_width_ = FLT_MAX; - float output_max_height_ = FLT_MAX; -#if !MEDIAPIPE_DISABLE_GPU - bool gpu_initialized_ = false; - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint program_ = 0; -#endif // !MEDIAPIPE_DISABLE_GPU -}; - -} // namespace mediapipe -#endif // MEDIAPIPE_CALCULATORS_IMAGE_IMAGE_CROPPING_CALCULATOR_H_ diff --git a/mediapipe/calculators/image/image_cropping_calculator.proto b/mediapipe/calculators/image/image_cropping_calculator.proto deleted file mode 100644 index 55d3467d1..000000000 --- a/mediapipe/calculators/image/image_cropping_calculator.proto +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message ImageCroppingCalculatorOptions { - extend CalculatorOptions { - optional ImageCroppingCalculatorOptions ext = 262466399; - } - - // Output texture buffer dimensions. The values defined in the options will be - // overriden by the WIDTH and HEIGHT input streams if they exist. - optional int32 width = 1; - optional int32 height = 2; - - // Rotation angle is counter-clockwise in radian. - optional float rotation = 3 [default = 0.0]; - - // Normalized width and height of the output rect. Value is within [0, 1]. - optional float norm_width = 4; - optional float norm_height = 5; - - // Normalized location of the center of the output - // rectangle in image coordinates. Value is within [0, 1]. - // The (0, 0) point is at the (top, left) corner. - optional float norm_center_x = 6 [default = 0]; - optional float norm_center_y = 7 [default = 0]; - - enum BorderMode { - // First unspecified value is required by the guideline. See details here: - // https://developers.google.com/protocol-buffers/docs/style#enums - BORDER_UNSPECIFIED = 0; - BORDER_ZERO = 1; - BORDER_REPLICATE = 2; - } - - // Specifies behaviour for crops that go beyond image borders. - optional BorderMode border_mode = 8 [default = BORDER_ZERO]; - - // Specifies limits for the size of the output image. It will be scaled down, - // preserving ratio, to fit within. These do not change which area of the - // input is selected for cropping. - optional int32 output_max_width = 9; - optional int32 output_max_height = 10; -} diff --git a/mediapipe/calculators/image/image_cropping_calculator_test.cc b/mediapipe/calculators/image/image_cropping_calculator_test.cc deleted file mode 100644 index b3f692889..000000000 --- a/mediapipe/calculators/image/image_cropping_calculator_test.cc +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/image_cropping_calculator.h" - -#include -#include - -#include "mediapipe/calculators/image/image_cropping_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/tag_map.h" -#include "mediapipe/framework/tool/tag_map_helper.h" - -namespace mediapipe { - -namespace { - -constexpr int input_width = 100; -constexpr int input_height = 100; - -constexpr char kRectTag[] = "RECT"; -constexpr char kHeightTag[] = "HEIGHT"; -constexpr char kWidthTag[] = "WIDTH"; - -// Test normal case, where norm_width and norm_height in options are set. -TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) { - auto calculator_node = - ParseTextProtoOrDie( - R"pb( - calculator: "ImageCroppingCalculator" - input_stream: "IMAGE_GPU:input_frames" - output_stream: "IMAGE_GPU:cropped_output_frames" - options: { - [mediapipe.ImageCroppingCalculatorOptions.ext] { - norm_width: 0.6 - norm_height: 0.6 - norm_center_x: 0.5 - norm_center_y: 0.5 - rotation: 0.3 - } - } - )pb"); - - auto calculator_state = absl::make_unique( - "Node", 0, "Calculator", calculator_node, nullptr); - auto cc = absl::make_unique( - calculator_state.get(), tool::CreateTagMap({}).value(), - tool::CreateTagMap({}).value()); - - RectSpec expectRect = { - .width = 60, - .height = 60, - .center_x = 50, - .center_y = 50, - .rotation = 0.3, - }; - EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, - input_height), - expectRect); -} // TEST - -// Test when (width height) + (norm_width norm_height) are set in options. -// width and height should take precedence. -TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) { - auto calculator_node = - ParseTextProtoOrDie( - R"pb( - calculator: "ImageCroppingCalculator" - input_stream: "IMAGE_GPU:input_frames" - output_stream: "IMAGE_GPU:cropped_output_frames" - options: { - [mediapipe.ImageCroppingCalculatorOptions.ext] { - width: 50 - height: 50 - norm_width: 0.6 - norm_height: 0.6 - norm_center_x: 0.5 - norm_center_y: 0.5 - rotation: 0.3 - } - } - )pb"); - - auto calculator_state = absl::make_unique( - "Node", 0, "Calculator", calculator_node, nullptr); - auto cc = absl::make_unique( - calculator_state.get(), tool::CreateTagMap({}).value(), - tool::CreateTagMap({}).value()); - RectSpec expectRect = { - .width = 50, - .height = 50, - .center_x = 50, - .center_y = 50, - .rotation = 0.3, - }; - EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, - input_height), - expectRect); -} // TEST - -// Test when WIDTH HEIGHT are set from input stream, -// and options has norm_width/height set. -// WIDTH HEIGHT from input stream should take precedence. -TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) { - auto calculator_node = - ParseTextProtoOrDie( - R"pb( - calculator: "ImageCroppingCalculator" - input_stream: "IMAGE_GPU:input_frames" - input_stream: "WIDTH:crop_width" - input_stream: "HEIGHT:crop_height" - output_stream: "IMAGE_GPU:cropped_output_frames" - options: { - [mediapipe.ImageCroppingCalculatorOptions.ext] { - width: 50 - height: 50 - norm_width: 0.6 - norm_height: 0.6 - norm_center_x: 0.5 - norm_center_y: 0.5 - rotation: 0.3 - } - } - )pb"); - - auto calculator_state = absl::make_unique( - "Node", 0, "Calculator", calculator_node, nullptr); - auto inputTags = tool::CreateTagMap({ - "HEIGHT:0:crop_height", - "WIDTH:0:crop_width", - }) - .value(); - auto cc = absl::make_unique( - calculator_state.get(), inputTags, tool::CreateTagMap({}).value()); - auto& inputs = cc->Inputs(); - inputs.Tag(kHeightTag).Value() = MakePacket(1); - inputs.Tag(kWidthTag).Value() = MakePacket(1); - RectSpec expectRect = { - .width = 1, - .height = 1, - .center_x = 50, - .center_y = 50, - .rotation = 0.3, - }; - EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, - input_height), - expectRect); -} // TEST - -// Test when RECT is set from input stream, -// and options has norm_width/height set. -// RECT from input stream should take precedence. -TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) { - auto calculator_node = - ParseTextProtoOrDie( - R"pb( - calculator: "ImageCroppingCalculator" - input_stream: "IMAGE_GPU:input_frames" - input_stream: "RECT:rect" - output_stream: "IMAGE_GPU:cropped_output_frames" - options: { - [mediapipe.ImageCroppingCalculatorOptions.ext] { - width: 50 - height: 50 - norm_width: 0.6 - norm_height: 0.6 - norm_center_x: 0.5 - norm_center_y: 0.5 - rotation: 0.3 - } - } - )pb"); - - auto calculator_state = absl::make_unique( - "Node", 0, "Calculator", calculator_node, nullptr); - auto inputTags = tool::CreateTagMap({ - "RECT:0:rect", - }) - .value(); - auto cc = absl::make_unique( - calculator_state.get(), inputTags, tool::CreateTagMap({}).value()); - auto& inputs = cc->Inputs(); - mediapipe::Rect rect = ParseTextProtoOrDie( - R"pb( - width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5 - )pb"); - inputs.Tag(kRectTag).Value() = MakePacket(rect); - RectSpec expectRect = { - .width = 1, - .height = 1, - .center_x = 40, - .center_y = 40, - .rotation = 0.5, - }; - EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, - input_height), - expectRect); -} // TEST - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_file_properties_calculator.cc b/mediapipe/calculators/image/image_file_properties_calculator.cc deleted file mode 100644 index 9c6d8caca..000000000 --- a/mediapipe/calculators/image/image_file_properties_calculator.cc +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "exif.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_file_properties.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace { - -// 35 MM sensor has dimensions 36 mm x 24 mm, so diagonal length is -// sqrt(36^2 + 24^2). -static const double SENSOR_DIAGONAL_35MM = std::sqrt(1872.0); - -absl::StatusOr ComputeFocalLengthInPixels(int image_width, - int image_height, - double focal_length_35mm, - double focal_length_mm) { - // TODO: Allow returning image file properties even when focal length - // computation is not possible. - if (image_width == 0 || image_height == 0) { - return absl::InternalError( - "Image dimensions should be non-zero to compute focal length in " - "pixels."); - } - if (focal_length_mm == 0) { - return absl::InternalError( - "Focal length in mm should be non-zero to compute focal length in " - "pixels."); - } - if (focal_length_35mm == 0) { - return absl::InternalError( - "Focal length in 35 mm should be non-zero to compute focal length in " - "pixels."); - } - // Derived from - // https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length#Calculation. - /// Using focal_length_35mm = focal_length_mm * SENSOR_DIAGONAL_35MM / - /// sensor_diagonal_mm, we can calculate the diagonal length of the sensor in - /// millimeters i.e. sensor_diagonal_mm. - double sensor_diagonal_mm = - SENSOR_DIAGONAL_35MM / focal_length_35mm * focal_length_mm; - // Note that for the following computations, the longer dimension is treated - // as image width and the shorter dimension is treated as image height. - int width = image_width; - int height = image_height; - if (image_height > image_width) { - width = image_height; - height = image_width; - } - double inv_aspect_ratio = (double)height / width; - // Compute sensor width. - /// Using Pythagoras theorem, sensor_width^2 + sensor_height^2 = - /// sensor_diagonal_mm^2. We can substitute sensor_width / sensor_height with - /// the aspect ratio calculated in pixels to compute the sensor width. - double sensor_width = std::sqrt((sensor_diagonal_mm * sensor_diagonal_mm) / - (1.0 + inv_aspect_ratio * inv_aspect_ratio)); - - // Compute focal length in pixels. - double focal_length_pixels = width * focal_length_mm / sensor_width; - return focal_length_pixels; -} - -absl::StatusOr GetImageFileProperites( - const std::string& image_bytes) { - easyexif::EXIFInfo result; - int code = result.parseFrom(image_bytes); - if (code) { - return absl::InternalError("Error parsing EXIF, code: " + - std::to_string(code)); - } - - ImageFileProperties properties; - properties.set_image_width(result.ImageWidth); - properties.set_image_height(result.ImageHeight); - properties.set_focal_length_mm(result.FocalLength); - properties.set_focal_length_35mm(result.FocalLengthIn35mm); - - ASSIGN_OR_RETURN(auto focal_length_pixels, - ComputeFocalLengthInPixels(properties.image_width(), - properties.image_height(), - properties.focal_length_35mm(), - properties.focal_length_mm())); - properties.set_focal_length_pixels(focal_length_pixels); - - return properties; -} - -} // namespace - -// Calculator to extract EXIF information from an image file. The input is -// a std::string containing raw byte data from a file, and the output is an -// ImageFileProperties proto object with the relevant fields filled in. -// The calculator accepts the input as a stream or a side packet, and can output -// the result as a stream or a side packet. The calculator checks that if an -// output stream is present, it outputs to that stream, and if not, it checks if -// it can output to a side packet. -// -// Example config with input and output streams: -// node { -// calculator: "ImageFilePropertiesCalculator" -// input_stream: "image_bytes" -// output_stream: "image_properties" -// } -// Example config with input and output side packets: -// node { -// calculator: "ImageFilePropertiesCalculator" -// input_side_packet: "image_bytes" -// output_side_packet: "image_properties" -// } -class ImageFilePropertiesCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - if (cc->Inputs().NumEntries() != 0) { - RET_CHECK(cc->Inputs().NumEntries() == 1); - cc->Inputs().Index(0).Set(); - } else { - RET_CHECK(cc->InputSidePackets().NumEntries() == 1); - cc->InputSidePackets().Index(0).Set(); - } - if (cc->Outputs().NumEntries() != 0) { - RET_CHECK(cc->Outputs().NumEntries() == 1); - cc->Outputs().Index(0).Set<::mediapipe::ImageFileProperties>(); - } else { - RET_CHECK(cc->OutputSidePackets().NumEntries() == 1); - cc->OutputSidePackets().Index(0).Set<::mediapipe::ImageFileProperties>(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - if (cc->InputSidePackets().NumEntries() == 1) { - const std::string& image_bytes = - cc->InputSidePackets().Index(0).Get(); - ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); - read_properties_ = true; - } - - if (read_properties_ && cc->OutputSidePackets().NumEntries() == 1) { - cc->OutputSidePackets().Index(0).Set( - MakePacket(properties_)); - } - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (cc->Inputs().NumEntries() == 1) { - if (cc->Inputs().Index(0).IsEmpty()) { - return absl::OkStatus(); - } - const std::string& image_bytes = cc->Inputs().Index(0).Get(); - ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); - read_properties_ = true; - } - if (read_properties_) { - if (cc->Outputs().NumEntries() == 1) { - cc->Outputs().Index(0).AddPacket( - MakePacket(properties_) - .At(cc->InputTimestamp())); - } else { - cc->OutputSidePackets().Index(0).Set( - MakePacket(properties_) - .At(mediapipe::Timestamp::Unset())); - } - } - - return absl::OkStatus(); - } - - private: - ImageFileProperties properties_; - bool read_properties_ = false; -}; -REGISTER_CALCULATOR(ImageFilePropertiesCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_file_properties_calculator_test.cc b/mediapipe/calculators/image/image_file_properties_calculator_test.cc deleted file mode 100644 index 5c973030f..000000000 --- a/mediapipe/calculators/image/image_file_properties_calculator_test.cc +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_file_properties.pb.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -constexpr char kImageFilePath[] = - "/mediapipe/calculators/image/testdata/" - "front_camera_pixel2.jpg"; -constexpr int kExpectedWidth = 2448; -constexpr int kExpectedHeight = 3264; -constexpr double kExpectedFocalLengthMm = 3.38; -constexpr double kExpectedFocalLengthIn35Mm = 25; -constexpr double kExpectedFocalLengthPixels = 2357.48; - -double RoundToNDecimals(double value, int n) { - return std::round(value * pow(10.0, n)) / pow(10.0, n); -} - -TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInStreams) { - std::string image_filepath = file::JoinPath("./", kImageFilePath); - std::string image_contents; - MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents)); - - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "ImageFilePropertiesCalculator" - input_stream: "image_bytes" - output_stream: "properties" - )pb"); - - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).packets.push_back( - MakePacket(image_contents).At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.Outputs(); - ASSERT_EQ(1, outputs.NumEntries()); - const std::vector& packets = outputs.Index(0).packets; - ASSERT_EQ(1, packets.size()); - const auto& result = packets[0].Get<::mediapipe::ImageFileProperties>(); - EXPECT_EQ(kExpectedWidth, result.image_width()); - EXPECT_EQ(kExpectedHeight, result.image_height()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels, - RoundToNDecimals(result.focal_length_pixels(), /*n=*/2)); -} - -TEST(ImageFilePropertiesCalculatorTest, ReadsFocalLengthFromJpegInSidePackets) { - std::string image_filepath = file::JoinPath("./", kImageFilePath); - std::string image_contents; - MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents)); - - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "ImageFilePropertiesCalculator" - input_side_packet: "image_bytes" - output_side_packet: "properties" - )pb"); - - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Index(0) = - MakePacket(image_contents).At(Timestamp(0)); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.OutputSidePackets(); - EXPECT_EQ(1, outputs.NumEntries()); - const auto& packet = outputs.Index(0); - const auto& result = packet.Get<::mediapipe::ImageFileProperties>(); - EXPECT_EQ(kExpectedWidth, result.image_width()); - EXPECT_EQ(kExpectedHeight, result.image_height()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels, - RoundToNDecimals(result.focal_length_pixels(), /*n=*/2)); -} - -TEST(ImageFilePropertiesCalculatorTest, - ReadsFocalLengthFromJpegStreamToSidePacket) { - std::string image_filepath = file::JoinPath("./", kImageFilePath); - std::string image_contents; - MP_ASSERT_OK(file::GetContents(image_filepath, &image_contents)); - - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "ImageFilePropertiesCalculator" - input_stream: "image_bytes" - output_side_packet: "properties" - )pb"); - - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).packets.push_back( - MakePacket(image_contents).At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.OutputSidePackets(); - EXPECT_EQ(1, outputs.NumEntries()); - const auto& packet = outputs.Index(0); - const auto& result = packet.Get<::mediapipe::ImageFileProperties>(); - EXPECT_EQ(kExpectedWidth, result.image_width()); - EXPECT_EQ(kExpectedHeight, result.image_height()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthMm, result.focal_length_mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthIn35Mm, result.focal_length_35mm()); - EXPECT_DOUBLE_EQ(kExpectedFocalLengthPixels, - RoundToNDecimals(result.focal_length_pixels(), /*n=*/2)); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_properties_calculator.cc b/mediapipe/calculators/image/image_properties_calculator.cc deleted file mode 100644 index 59011804e..000000000 --- a/mediapipe/calculators/image/image_properties_calculator.cc +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gpu_buffer.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { -namespace api2 { - -#if MEDIAPIPE_DISABLE_GPU -// Just a placeholder to not have to depend on mediapipe::GpuBuffer. -using GpuBuffer = AnyType; -#else -using GpuBuffer = mediapipe::GpuBuffer; -#endif // MEDIAPIPE_DISABLE_GPU - -// Extracts image properties from the input image and outputs the properties. -// Currently only supports image size. -// Input: -// One of the following: -// IMAGE: An Image or ImageFrame (for backward compatibility with existing -// graphs that use IMAGE for ImageFrame input) -// IMAGE_CPU: An ImageFrame -// IMAGE_GPU: A GpuBuffer -// -// Output: -// SIZE: Size (as a std::pair) of the input image. -// -// Example usage: -// node { -// calculator: "ImagePropertiesCalculator" -// input_stream: "IMAGE:image" -// output_stream: "SIZE:size" -// } -class ImagePropertiesCalculator : public Node { - public: - static constexpr Input< - OneOf>::Optional kIn{"IMAGE"}; - // IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level - // graphs for the Python Solution APIs to figure out the type of input stream - // without running into ambiguities from IMAGE. - // TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image. - static constexpr Input::Optional kInCpu{"IMAGE_CPU"}; - static constexpr Input::Optional kInGpu{"IMAGE_GPU"}; - static constexpr Output> kOut{"SIZE"}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut); - - static absl::Status UpdateContract(CalculatorContract* cc) { - RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() + - kInGpu(cc).IsConnected(), - 1) - << "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is " - "expected."; - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - std::pair size; - - if (kIn(cc).IsConnected()) { - kIn(cc).Visit( - [&size](const mediapipe::Image& value) { - size.first = value.width(); - size.second = value.height(); - }, - [&size](const mediapipe::ImageFrame& value) { - size.first = value.Width(); - size.second = value.Height(); - }); - } - if (kInCpu(cc).IsConnected()) { - const auto& image = *kInCpu(cc); - size.first = image.Width(); - size.second = image.Height(); - } -#if !MEDIAPIPE_DISABLE_GPU - if (kInGpu(cc).IsConnected()) { - const auto& image = *kInGpu(cc); - size.first = image.width(); - size.second = image.height(); - } -#endif // !MEDIAPIPE_DISABLE_GPU - - kOut(cc).Send(size); - - return absl::OkStatus(); - } -}; - -MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_transformation_calculator.cc b/mediapipe/calculators/image/image_transformation_calculator.cc deleted file mode 100644 index 60873ae9f..000000000 --- a/mediapipe/calculators/image/image_transformation_calculator.cc +++ /dev/null @@ -1,612 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/image_transformation_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/scale_mode.pb.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_quad_renderer.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -#if defined(__ANDROID__) -// The size of Java arrays is dynamic, which makes it difficult to -// generate the right packet type with a fixed size. Therefore, we -// are using unsized arrays on Android. -typedef int DimensionsPacketType[]; -#else -typedef int DimensionsPacketType[2]; -#endif // __ANDROID__ - -#define DEFAULT_SCALE_MODE mediapipe::ScaleMode_Mode_STRETCH - -namespace mediapipe { - -#if !MEDIAPIPE_DISABLE_GPU - -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace { -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; - -int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) { - switch (rotation) { - case mediapipe::RotationMode_Mode_UNKNOWN: - case mediapipe::RotationMode_Mode_ROTATION_0: - return 0; - case mediapipe::RotationMode_Mode_ROTATION_90: - return 90; - case mediapipe::RotationMode_Mode_ROTATION_180: - return 180; - case mediapipe::RotationMode_Mode_ROTATION_270: - return 270; - } -} -mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) { - switch (degrees) { - case 0: - return mediapipe::RotationMode_Mode_ROTATION_0; - case 90: - return mediapipe::RotationMode_Mode_ROTATION_90; - case 180: - return mediapipe::RotationMode_Mode_ROTATION_180; - case 270: - return mediapipe::RotationMode_Mode_ROTATION_270; - default: - return mediapipe::RotationMode_Mode_UNKNOWN; - } -} -mediapipe::ScaleMode_Mode ParseScaleMode( - mediapipe::ScaleMode_Mode scale_mode, - mediapipe::ScaleMode_Mode default_mode) { - switch (scale_mode) { - case mediapipe::ScaleMode_Mode_DEFAULT: - return default_mode; - case mediapipe::ScaleMode_Mode_STRETCH: - return scale_mode; - case mediapipe::ScaleMode_Mode_FIT: - return scale_mode; - case mediapipe::ScaleMode_Mode_FILL_AND_CROP: - return scale_mode; - default: - return default_mode; - } -} -} // namespace - -// Scales, rotates, and flips images horizontally or vertically. -// -// Input: -// One of the following tags: -// IMAGE: ImageFrame representing the input image. -// IMAGE_GPU: GpuBuffer representing the input image. -// -// ROTATION_DEGREES (optional): The counterclockwise rotation angle in -// degrees. This allows different rotation angles for different frames. It has -// to be a multiple of 90 degrees. If provided, it overrides the -// ROTATION_DEGREES input side packet. -// -// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If -// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or -// corresponding field in the calculator options. -// -// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If -// provided, it overrides the FLIP_VERTICALLY input side packet and/or -// corresponding field in the calculator options. -// -// Output: -// One of the following tags: -// IMAGE - ImageFrame representing the output image. -// IMAGE_GPU - GpuBuffer representing the output image. -// -// LETTERBOX_PADDING (optional): An std::array representing the -// letterbox padding from the 4 sides ([left, top, right, bottom]) of the -// output image, normalized to [0.f, 1.f] by the output dimensions. The -// padding values are non-zero only when the scale mode specified in the -// calculator options is FIT. For instance, when the input image is 10x10 -// (width x height) and the output dimensions specified in the calculator -// option are 20x40 and scale mode is FIT, the calculator scales the input -// image to 20x20 and places it in the middle of the output image with an -// equal padding of 10 pixels at the top and the bottom. The resulting array -// is therefore [0.f, 0.25f, 0.f, 0.25f] (10/40 = 0.25f). -// -// Input side packet: -// OUTPUT_DIMENSIONS (optional): The output width and height in pixels as the -// first two elements in an integer array. It overrides the corresponding -// field in the calculator options. -// -// ROTATION_DEGREES (optional): The counterclockwise rotation angle in -// degrees. It has to be a multiple of 90 degrees. It overrides the -// corresponding field in the calculator options. -// -// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. -// It overrides the corresponding field in the calculator options. -// -// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. -// It overrides the corresponding field in the calculator options. -// -// Calculator options (see image_transformation_calculator.proto): -// output_width, output_height - (optional) Desired scaled image size. -// rotation_mode - (optional) Rotation in multiples of 90 degrees. -// flip_vertically, flip_horizontally - (optional) flip about x or y axis. -// scale_mode - (optional) Stretch, Fit, or Fill and Crop -// -// Note: To enable horizontal or vertical flipping, specify them in the -// calculator options. Flipping is applied after rotation. -// -// Note: Input defines output, so only matchig types supported: -// IMAGE -> IMAGE or IMAGE_GPU -> IMAGE_GPU -// -class ImageTransformationCalculator : public CalculatorBase { - public: - ImageTransformationCalculator() = default; - ~ImageTransformationCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status RenderCpu(CalculatorContext* cc); - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status GlSetup(); - - void ComputeOutputDimensions(int input_width, int input_height, - int* output_width, int* output_height); - void ComputeOutputLetterboxPadding(int input_width, int input_height, - int output_width, int output_height, - std::array* padding); - - ImageTransformationCalculatorOptions options_; - int output_width_ = 0; - int output_height_ = 0; - mediapipe::RotationMode_Mode rotation_; - mediapipe::ScaleMode_Mode scale_mode_; - bool flip_horizontally_ = false; - bool flip_vertically_ = false; - - bool use_gpu_ = false; -#if !MEDIAPIPE_DISABLE_GPU - GlCalculatorHelper gpu_helper_; - std::unique_ptr rgb_renderer_; - std::unique_ptr yuv_renderer_; - std::unique_ptr ext_rgb_renderer_; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(ImageTransformationCalculator); - -// static -absl::Status ImageTransformationCalculator::GetContract( - CalculatorContract* cc) { - // Only one input can be set, and the output type must match. - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^ - cc->Inputs().HasTag(kGpuBufferTag)); - - bool use_gpu = false; - - if (cc->Inputs().HasTag(kImageFrameTag)) { - RET_CHECK(cc->Outputs().HasTag(kImageFrameTag)); - cc->Inputs().Tag(kImageFrameTag).Set(); - cc->Outputs().Tag(kImageFrameTag).Set(); - } -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag)); - cc->Inputs().Tag(kGpuBufferTag).Set(); - cc->Outputs().Tag(kGpuBufferTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - - if (cc->Inputs().HasTag("ROTATION_DEGREES")) { - cc->Inputs().Tag("ROTATION_DEGREES").Set(); - } - if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) { - cc->Inputs().Tag("FLIP_HORIZONTALLY").Set(); - } - if (cc->Inputs().HasTag("FLIP_VERTICALLY")) { - cc->Inputs().Tag("FLIP_VERTICALLY").Set(); - } - - if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) { - cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set(); - } - if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { - cc->InputSidePackets().Tag("ROTATION_DEGREES").Set(); - } - if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { - cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set(); - } - if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { - cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set(); - } - - if (cc->Outputs().HasTag("LETTERBOX_PADDING")) { - cc->Outputs().Tag("LETTERBOX_PADDING").Set>(); - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) { - // Inform the framework that we always output at the same timestamp - // as we receive a packet at. - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - - if (cc->Inputs().HasTag(kGpuBufferTag)) { - use_gpu_ = true; - } - - if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) { - const auto& dimensions = cc->InputSidePackets() - .Tag("OUTPUT_DIMENSIONS") - .Get(); - output_width_ = dimensions[0]; - output_height_ = dimensions[1]; - } else { - output_width_ = options_.output_width(); - output_height_ = options_.output_height(); - } - - if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { - rotation_ = DegreesToRotationMode( - cc->InputSidePackets().Tag("ROTATION_DEGREES").Get()); - } else { - rotation_ = options_.rotation_mode(); - } - - if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { - flip_horizontally_ = - cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get(); - } else { - flip_horizontally_ = options_.flip_horizontally(); - } - - if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { - flip_vertically_ = - cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get(); - } else { - flip_vertically_ = options_.flip_vertically(); - } - - scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE); - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - // Let the helper access the GL context information. - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#else - RET_CHECK_FAIL() << "GPU processing not enabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status ImageTransformationCalculator::Process(CalculatorContext* cc) { - // Override values if specified so. - if (cc->Inputs().HasTag("ROTATION_DEGREES") && - !cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) { - rotation_ = - DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get()); - } - if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") && - !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) { - flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get(); - } - if (cc->Inputs().HasTag("FLIP_VERTICALLY") && - !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) { - flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get(); - } - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) { - return absl::OkStatus(); - } - return gpu_helper_.RunInGlContext( - [this, cc]() -> absl::Status { return RenderGpu(cc); }); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - return RenderCpu(cc); - } - return absl::OkStatus(); -} - -absl::Status ImageTransformationCalculator::Close(CalculatorContext* cc) { - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - QuadRenderer* rgb_renderer = rgb_renderer_.release(); - QuadRenderer* yuv_renderer = yuv_renderer_.release(); - QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release(); - gpu_helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] { - if (rgb_renderer) { - rgb_renderer->GlTeardown(); - delete rgb_renderer; - } - if (ext_rgb_renderer) { - ext_rgb_renderer->GlTeardown(); - delete ext_rgb_renderer; - } - if (yuv_renderer) { - yuv_renderer->GlTeardown(); - delete yuv_renderer; - } - }); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) { - cv::Mat input_mat; - mediapipe::ImageFormat::Format format; - - const auto& input = cc->Inputs().Tag(kImageFrameTag).Get(); - input_mat = formats::MatView(&input); - format = input.Format(); - - const int input_width = input_mat.cols; - const int input_height = input_mat.rows; - int output_width; - int output_height; - ComputeOutputDimensions(input_width, input_height, &output_width, - &output_height); - - if (output_width_ > 0 && output_height_ > 0) { - cv::Mat scaled_mat; - if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) { - int scale_flag = - input_mat.cols > output_width_ && input_mat.rows > output_height_ - ? cv::INTER_AREA - : cv::INTER_LINEAR; - cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_), - 0, 0, scale_flag); - } else { - const float scale = - std::min(static_cast(output_width_) / input_width, - static_cast(output_height_) / input_height); - const int target_width = std::round(input_width * scale); - const int target_height = std::round(input_height * scale); - int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR; - if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) { - cv::Mat intermediate_mat; - cv::resize(input_mat, intermediate_mat, - cv::Size(target_width, target_height), 0, 0, scale_flag); - const int top = (output_height_ - target_height) / 2; - const int bottom = output_height_ - target_height - top; - const int left = (output_width_ - target_width) / 2; - const int right = output_width_ - target_width - left; - cv::copyMakeBorder(intermediate_mat, scaled_mat, top, bottom, left, - right, - options_.constant_padding() ? cv::BORDER_CONSTANT - : cv::BORDER_REPLICATE); - } else { - cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height), - 0, 0, scale_flag); - output_width = target_width; - output_height = target_height; - } - } - input_mat = scaled_mat; - } - - if (cc->Outputs().HasTag("LETTERBOX_PADDING")) { - auto padding = absl::make_unique>(); - ComputeOutputLetterboxPadding(input_width, input_height, output_width, - output_height, padding.get()); - cc->Outputs() - .Tag("LETTERBOX_PADDING") - .Add(padding.release(), cc->InputTimestamp()); - } - - cv::Mat rotated_mat; - cv::Size rotated_size(output_width, output_height); - if (input_mat.size() == rotated_size) { - const int angle = RotationModeToDegrees(rotation_); - cv::Point2f src_center(input_mat.cols / 2.0, input_mat.rows / 2.0); - cv::Mat rotation_mat = cv::getRotationMatrix2D(src_center, angle, 1.0); - cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size); - } else { - switch (rotation_) { - case mediapipe::RotationMode_Mode_UNKNOWN: - case mediapipe::RotationMode_Mode_ROTATION_0: - rotated_mat = input_mat; - break; - case mediapipe::RotationMode_Mode_ROTATION_90: - cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE); - break; - case mediapipe::RotationMode_Mode_ROTATION_180: - cv::rotate(input_mat, rotated_mat, cv::ROTATE_180); - break; - case mediapipe::RotationMode_Mode_ROTATION_270: - cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE); - break; - } - } - - cv::Mat flipped_mat; - if (flip_horizontally_ || flip_vertically_) { - const int flip_code = - flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_; - cv::flip(rotated_mat, flipped_mat, flip_code); - } else { - flipped_mat = rotated_mat; - } - - std::unique_ptr output_frame( - new ImageFrame(format, output_width, output_height)); - cv::Mat output_mat = formats::MatView(output_frame.get()); - flipped_mat.copyTo(output_mat); - cc->Outputs() - .Tag(kImageFrameTag) - .Add(output_frame.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get(); - const int input_width = input.width(); - const int input_height = input.height(); - - int output_width; - int output_height; - ComputeOutputDimensions(input_width, input_height, &output_width, - &output_height); - - if (cc->Outputs().HasTag("LETTERBOX_PADDING")) { - auto padding = absl::make_unique>(); - ComputeOutputLetterboxPadding(input_width, input_height, output_width, - output_height, padding.get()); - cc->Outputs() - .Tag("LETTERBOX_PADDING") - .Add(padding.release(), cc->InputTimestamp()); - } - - QuadRenderer* renderer = nullptr; - GlTexture src1; - -#if defined(MEDIAPIPE_IOS) - if (input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange || - input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) { - if (!yuv_renderer_) { - yuv_renderer_ = absl::make_unique(); - MP_RETURN_IF_ERROR( - yuv_renderer_->GlSetup(::mediapipe::kYUV2TexToRGBFragmentShader, - {"video_frame_y", "video_frame_uv"})); - } - renderer = yuv_renderer_.get(); - src1 = gpu_helper_.CreateSourceTexture(input, 0); - } else // NOLINT(readability/braces) -#endif // iOS - { - src1 = gpu_helper_.CreateSourceTexture(input); -#if defined(TEXTURE_EXTERNAL_OES) - if (src1.target() == GL_TEXTURE_EXTERNAL_OES) { - if (!ext_rgb_renderer_) { - ext_rgb_renderer_ = absl::make_unique(); - MP_RETURN_IF_ERROR(ext_rgb_renderer_->GlSetup( - ::mediapipe::kBasicTexturedFragmentShaderOES, {"video_frame"})); - } - renderer = ext_rgb_renderer_.get(); - } else // NOLINT(readability/braces) -#endif // TEXTURE_EXTERNAL_OES - { - if (!rgb_renderer_) { - rgb_renderer_ = absl::make_unique(); - MP_RETURN_IF_ERROR(rgb_renderer_->GlSetup()); - } - renderer = rgb_renderer_.get(); - } - } - RET_CHECK(renderer) << "Unsupported input texture type"; - - mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto( - scale_mode_, mediapipe::FrameScaleMode::kStretch); - mediapipe::FrameRotation rotation = - mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_)); - - auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height, - input.format()); - - gpu_helper_.BindFramebuffer(dst); - glActiveTexture(GL_TEXTURE1); - glBindTexture(src1.target(), src1.name()); - - MP_RETURN_IF_ERROR(renderer->GlRender( - src1.width(), src1.height(), dst.width(), dst.height(), scale_mode, - rotation, flip_horizontally_, flip_vertically_, - /*flip_texture=*/false)); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(src1.target(), 0); - - // Execute GL commands, before getting result. - glFlush(); - - auto output = dst.template GetFrame(); - cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp()); - -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void ImageTransformationCalculator::ComputeOutputDimensions( - int input_width, int input_height, int* output_width, int* output_height) { - if (output_width_ > 0 && output_height_ > 0) { - *output_width = output_width_; - *output_height = output_height_; - } else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 || - rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) { - *output_width = input_height; - *output_height = input_width; - } else { - *output_width = input_width; - *output_height = input_height; - } -} - -void ImageTransformationCalculator::ComputeOutputLetterboxPadding( - int input_width, int input_height, int output_width, int output_height, - std::array* padding) { - padding->fill(0.f); - if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) { - if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 || - rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) { - std::swap(input_width, input_height); - } - const float input_aspect_ratio = - static_cast(input_width) / input_height; - const float output_aspect_ratio = - static_cast(output_width) / output_height; - if (input_aspect_ratio < output_aspect_ratio) { - // Compute left and right padding. - (*padding)[0] = (1.f - input_aspect_ratio / output_aspect_ratio) / 2.f; - (*padding)[2] = (*padding)[0]; - } else if (output_aspect_ratio < input_aspect_ratio) { - // Compute top and bottom padding. - (*padding)[1] = (1.f - output_aspect_ratio / input_aspect_ratio) / 2.f; - (*padding)[3] = (*padding)[1]; - } - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/image_transformation_calculator.proto b/mediapipe/calculators/image/image_transformation_calculator.proto deleted file mode 100644 index c90e03be9..000000000 --- a/mediapipe/calculators/image/image_transformation_calculator.proto +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/gpu/scale_mode.proto"; - -// Counterclockwise rotation. -message RotationMode { - enum Mode { - UNKNOWN = 0; - ROTATION_0 = 1; - ROTATION_90 = 2; - ROTATION_180 = 3; - ROTATION_270 = 4; - } -} - -message ImageTransformationCalculatorOptions { - extend CalculatorOptions { - optional ImageTransformationCalculatorOptions ext = 251952830; - } - - // Output dimensions. Set to 0 if they should be the same as the input. - optional int32 output_width = 1 [default = 0]; - optional int32 output_height = 2 [default = 0]; - // Counterclockwise rotation mode. - optional RotationMode.Mode rotation_mode = 3; - // Vertical flipping, applied after rotation. - optional bool flip_vertically = 4 [default = false]; - // Horizontal flipping, applied after rotation. - optional bool flip_horizontally = 5 [default = false]; - // Scale mode. - optional ScaleMode.Mode scale_mode = 6; - // Padding type. This option is only used when the scale mode is FIT. - // Default is to use BORDER_CONSTANT. If set to false, it will use - // BORDER_REPLICATE instead. - optional bool constant_padding = 7 [default = true]; -} diff --git a/mediapipe/calculators/image/luminance_calculator.cc b/mediapipe/calculators/image/luminance_calculator.cc deleted file mode 100644 index d5122c7a4..000000000 --- a/mediapipe/calculators/image/luminance_calculator.cc +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/gl_simple_calculator.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -namespace mediapipe { - -// Converts RGB images into luminance images, still stored in RGB format. -// See GlSimpleCalculatorBase for inputs, outputs and input side packets. -class LuminanceCalculator : public GlSimpleCalculator { - public: - absl::Status GlSetup() override; - absl::Status GlRender(const GlTexture& src, const GlTexture& dst) override; - absl::Status GlTeardown() override; - - private: - GLuint program_ = 0; - GLint frame_; -}; -REGISTER_CALCULATOR(LuminanceCalculator); - -absl::Status LuminanceCalculator::GlSetup() { - // Load vertex and fragment shaders - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - const GLchar* frag_src = GLES_VERSION_COMPAT - R"( -#if __VERSION__ < 130 - #define in varying -#endif // __VERSION__ < 130 - -#ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; -#else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; -#endif // defined(GL_ES) - - in vec2 sample_coordinate; - uniform sampler2D video_frame; - const highp vec3 W = vec3(0.2125, 0.7154, 0.0721); - - void main() { - vec4 color = texture2D(video_frame, sample_coordinate); - float luminance = dot(color.rgb, W); - fragColor.rgb = vec3(luminance); - fragColor.a = color.a; - } - - )"; - - // shader program - GlhCreateProgram(kBasicVertexShader, frag_src, NUM_ATTRIBUTES, - (const GLchar**)&attr_name[0], attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - frame_ = glGetUniformLocation(program_, "video_frame"); - return absl::OkStatus(); -} - -absl::Status LuminanceCalculator::GlRender(const GlTexture& src, - const GlTexture& dst) { - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(program_); - glUniform1i(frame_, 1); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); - - return absl::OkStatus(); -} - -absl::Status LuminanceCalculator::GlTeardown() { - if (program_) { - glDeleteProgram(program_); - program_ = 0; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/mask_overlay_calculator.cc b/mediapipe/calculators/image/mask_overlay_calculator.cc deleted file mode 100644 index 5fbf9e4f4..000000000 --- a/mediapipe/calculators/image/mask_overlay_calculator.cc +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/mask_overlay_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -namespace mediapipe { - -using ::mediapipe::MaskOverlayCalculatorOptions_MaskChannel_ALPHA; -using ::mediapipe::MaskOverlayCalculatorOptions_MaskChannel_RED; -using ::mediapipe::MaskOverlayCalculatorOptions_MaskChannel_UNKNOWN; - -// Mixes two frames using a third mask frame or constant value. -// -// Inputs: -// VIDEO:[0,1] (GpuBuffer): -// Two inputs should be provided. -// MASK (GpuBuffer): -// Optional. -// Where the mask is 0, VIDEO:0 will be used. Where it is 1, VIDEO:1. -// Intermediate values will blend. -// If not specified, CONST_MASK float must be present. -// CONST_MASK (float): -// Optional. -// If not specified, MASK GpuBuffer must be present. -// Similar to MASK GpuBuffer, but applied globally to every pixel. -// -// Outputs: -// OUTPUT (GpuBuffer): -// The mix. - -class MaskOverlayCalculator : public CalculatorBase { - public: - MaskOverlayCalculator() {} - ~MaskOverlayCalculator(); - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - absl::Status GlSetup( - const MaskOverlayCalculatorOptions::MaskChannel mask_channel); - absl::Status GlRender(const float mask_const); - - private: - GlCalculatorHelper helper_; - bool initialized_ = false; - bool use_mask_tex_ = false; // Otherwise, use constant float value. - GLuint program_ = 0; - GLint unif_frame1_; - GLint unif_frame2_; - GLint unif_mask_; -}; -REGISTER_CALCULATOR(MaskOverlayCalculator); - -// static -absl::Status MaskOverlayCalculator::GetContract(CalculatorContract* cc) { - MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc)); - cc->Inputs().Get("VIDEO", 0).Set(); - cc->Inputs().Get("VIDEO", 1).Set(); - if (cc->Inputs().HasTag("MASK")) - cc->Inputs().Tag("MASK").Set(); - else if (cc->Inputs().HasTag("CONST_MASK")) - cc->Inputs().Tag("CONST_MASK").Set(); - else - return absl::Status(absl::StatusCode::kNotFound, - "At least one mask input stream must be present."); - cc->Outputs().Tag("OUTPUT").Set(); - return absl::OkStatus(); -} - -absl::Status MaskOverlayCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag("MASK")) { - use_mask_tex_ = true; - } - return helper_.Open(cc); -} - -absl::Status MaskOverlayCalculator::Process(CalculatorContext* cc) { - return helper_.RunInGlContext([this, &cc]() -> absl::Status { - if (!initialized_) { - const auto& options = cc->Options(); - const auto mask_channel = options.mask_channel(); - - MP_RETURN_IF_ERROR(GlSetup(mask_channel)); - initialized_ = true; - } - - glDisable(GL_BLEND); - - const Packet& input1_packet = cc->Inputs().Get("VIDEO", 1).Value(); - const Packet& mask_packet = use_mask_tex_ - ? cc->Inputs().Tag("MASK").Value() - : cc->Inputs().Tag("CONST_MASK").Value(); - - if (mask_packet.IsEmpty()) { - cc->Outputs().Tag("OUTPUT").AddPacket(input1_packet); - return absl::OkStatus(); - } - - const auto& input0_buffer = cc->Inputs().Get("VIDEO", 0).Get(); - const auto& input1_buffer = input1_packet.Get(); - - auto src1 = helper_.CreateSourceTexture(input0_buffer); - auto src2 = helper_.CreateSourceTexture(input1_buffer); - - GlTexture mask_tex; - if (use_mask_tex_) { - const auto& mask_buffer = mask_packet.Get(); - mask_tex = helper_.CreateSourceTexture(mask_buffer); - } - - auto dst = helper_.CreateDestinationTexture(src1.width(), src1.height()); - - helper_.BindFramebuffer(dst); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(src1.target(), src1.name()); - - glActiveTexture(GL_TEXTURE2); - glBindTexture(src2.target(), src2.name()); - - if (use_mask_tex_) { - const float mask_const = -1; - - glActiveTexture(GL_TEXTURE3); - glBindTexture(mask_tex.target(), mask_tex.name()); - - MP_RETURN_IF_ERROR(GlRender(mask_const)); - - glActiveTexture(GL_TEXTURE3); - glBindTexture(mask_tex.target(), 0); - - } else { - const float mask_const = mask_packet.Get(); - - MP_RETURN_IF_ERROR(GlRender(mask_const)); - } - - glActiveTexture(GL_TEXTURE2); - glBindTexture(src2.target(), 0); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(src1.target(), 0); - - glFlush(); - - auto output = dst.GetFrame(); - src1.Release(); - src2.Release(); - if (use_mask_tex_) mask_tex.Release(); - dst.Release(); - - cc->Outputs().Tag("OUTPUT").Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - }); -} - -absl::Status MaskOverlayCalculator::GlSetup( - const MaskOverlayCalculatorOptions::MaskChannel mask_channel) { - // Load vertex and fragment shaders - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - std::string mask_component; - switch (mask_channel) { - case MaskOverlayCalculatorOptions_MaskChannel_UNKNOWN: - case MaskOverlayCalculatorOptions_MaskChannel_RED: - mask_component = "r"; - break; - case MaskOverlayCalculatorOptions_MaskChannel_ALPHA: - mask_component = "a"; - break; - } - - const std::string frag_src_tex = - std::string(kMediaPipeFragmentShaderPreamble) + - R"( - DEFAULT_PRECISION(highp, float) - - in vec2 sample_coordinate; - uniform sampler2D frame1; - uniform sampler2D frame2; - uniform sampler2D mask; - - void main() { - vec4 color1 = texture2D(frame1, sample_coordinate); - vec4 color2 = texture2D(frame2, sample_coordinate); - vec4 weight = texture2D(mask, sample_coordinate); - - #define MASK_COMPONENT )" + - mask_component + - R"( - - gl_FragColor = mix(color1, color2, weight.MASK_COMPONENT); - } - )"; - - const GLchar* frag_src_const = R"( - precision highp float; - - varying vec2 sample_coordinate; - uniform sampler2D frame1; - uniform sampler2D frame2; - uniform float mask; - - void main() { - vec4 color1 = texture2D(frame1, sample_coordinate); - vec4 color2 = texture2D(frame2, sample_coordinate); - float weight = mask; - - gl_FragColor = mix(color1, color2, weight); - } - )"; - - // shader program - GlhCreateProgram(kBasicVertexShader, - use_mask_tex_ ? frag_src_tex.c_str() : frag_src_const, - NUM_ATTRIBUTES, &attr_name[0], attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - unif_frame1_ = glGetUniformLocation(program_, "frame1"); - unif_frame2_ = glGetUniformLocation(program_, "frame2"); - unif_mask_ = glGetUniformLocation(program_, "mask"); - return absl::OkStatus(); -} - -absl::Status MaskOverlayCalculator::GlRender(const float mask_const) { - glUseProgram(program_); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, kBasicSquareVertices); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, - kBasicTextureVertices); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - - glUniform1i(unif_frame1_, 1); - glUniform1i(unif_frame2_, 2); - if (use_mask_tex_) - glUniform1i(unif_mask_, 3); - else - glUniform1f(unif_mask_, mask_const); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - return absl::OkStatus(); -} - -MaskOverlayCalculator::~MaskOverlayCalculator() { - helper_.RunInGlContext([this] { - if (program_) { - glDeleteProgram(program_); - program_ = 0; - } - }); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/mask_overlay_calculator.proto b/mediapipe/calculators/image/mask_overlay_calculator.proto deleted file mode 100644 index 3b82d610e..000000000 --- a/mediapipe/calculators/image/mask_overlay_calculator.proto +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message MaskOverlayCalculatorOptions { - extend CalculatorOptions { - optional MaskOverlayCalculatorOptions ext = 252129282; - } - - enum MaskChannel { - UNKNOWN = 0; - RED = 1; - ALPHA = 2; - } - - // Selects which channel of the MASK input to use for masking. - optional MaskChannel mask_channel = 1 [default = RED]; -} diff --git a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc deleted file mode 100644 index 21bc587f3..000000000 --- a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { - -// Takes in an encoded image std::string, decodes it by OpenCV, and converts to -// an ImageFrame. Note that this calculator only supports grayscale and RGB -// images for now. -// -// Example config: -// node { -// calculator: "OpenCvEncodedImageToImageFrameCalculator" -// input_stream: "encoded_image" -// output_stream: "image_frame" -// } -class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_; -}; - -absl::Status OpenCvEncodedImageToImageFrameCalculator::GetContract( - CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); -} - -absl::Status OpenCvEncodedImageToImageFrameCalculator::Open( - CalculatorContext* cc) { - options_ = - cc->Options(); - return absl::OkStatus(); -} - -absl::Status OpenCvEncodedImageToImageFrameCalculator::Process( - CalculatorContext* cc) { - const std::string& contents = cc->Inputs().Index(0).Get(); - const std::vector contents_vector(contents.begin(), contents.end()); - cv::Mat decoded_mat; - if (options_.apply_orientation_from_exif_data()) { - // We want to respect the orientation from the EXIF data, which - // IMREAD_UNCHANGED ignores, but otherwise we want to be as permissive as - // possible with our reading flags. Therefore, we use IMREAD_ANYCOLOR and - // IMREAD_ANYDEPTH. - decoded_mat = cv::imdecode(contents_vector, - cv::IMREAD_ANYCOLOR | cv::IMREAD_ANYDEPTH); - } else { - // Return the loaded image as-is - decoded_mat = cv::imdecode(contents_vector, cv::IMREAD_UNCHANGED); - } - ImageFormat::Format image_format = ImageFormat::UNKNOWN; - cv::Mat output_mat; - switch (decoded_mat.channels()) { - case 1: - image_format = ImageFormat::GRAY8; - output_mat = decoded_mat; - break; - case 3: - image_format = ImageFormat::SRGB; - cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB); - break; - case 4: - image_format = ImageFormat::SRGBA; - cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGBA); - break; - default: - return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported number of channels: " << decoded_mat.channels(); - } - std::unique_ptr output_frame = absl::make_unique( - image_format, decoded_mat.size().width, decoded_mat.size().height, - ImageFrame::kGlDefaultAlignmentBoundary); - output_mat.copyTo(formats::MatView(output_frame.get())); - cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(OpenCvEncodedImageToImageFrameCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto deleted file mode 100644 index 3ec29bddf..000000000 --- a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenCvEncodedImageToImageFrameCalculatorOptions { - extend CalculatorOptions { - optional OpenCvEncodedImageToImageFrameCalculatorOptions ext = 303447308; - } - - // If set, we will attempt to automatically apply the orientation specified by - // the image's EXIF data when loading the image. Otherwise, the image data - // will be loaded as-is. - optional bool apply_orientation_from_exif_data = 1 [default = false]; -} diff --git a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc deleted file mode 100644 index 88e98ae07..000000000 --- a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator_test.cc +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestRgbJpeg) { - std::string contents; - MP_ASSERT_OK(file::GetContents( - file::JoinPath("./", "/mediapipe/calculators/image/testdata/dino.jpg"), - &contents)); - Packet input_packet = MakePacket(contents); - - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "OpenCvEncodedImageToImageFrameCalculator" - input_stream: "encoded_image" - output_stream: "image_frame" - )pb"); - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).packets.push_back( - input_packet.At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.Outputs(); - ASSERT_EQ(1, outputs.NumEntries()); - const std::vector& packets = outputs.Index(0).packets; - ASSERT_EQ(1, packets.size()); - const ImageFrame& output_frame = packets[0].Get(); - - cv::Mat input_mat = cv::imread( - file::JoinPath("./", "/mediapipe/calculators/image/testdata/dino.jpg")); - cv::Mat output_mat; - cv::cvtColor(formats::MatView(&output_frame), output_mat, cv::COLOR_RGB2BGR); - cv::Mat diff; - cv::absdiff(input_mat, output_mat, diff); - double max_val; - cv::minMaxLoc(diff, nullptr, &max_val); - // Expects that the maximum absolute pixel-by-pixel difference is less - // than 10. - EXPECT_LE(max_val, 10); -} - -TEST(OpenCvEncodedImageToImageFrameCalculatorTest, TestGrayscaleJpeg) { - cv::Mat input_mat; - cv::cvtColor(cv::imread(file::JoinPath("./", - "/mediapipe/calculators/" - "image/testdata/dino.jpg")), - input_mat, cv::COLOR_RGB2GRAY); - std::vector encode_buffer; - std::vector parameters; - parameters.push_back(cv::IMWRITE_JPEG_QUALITY); - parameters.push_back(100); - cv::imencode(".jpg", input_mat, encode_buffer, parameters); - Packet input_packet = MakePacket(std::string(absl::string_view( - reinterpret_cast(&encode_buffer[0]), encode_buffer.size()))); - - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "OpenCvEncodedImageToImageFrameCalculator" - input_stream: "encoded_image" - output_stream: "image_frame" - )pb"); - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).packets.push_back( - input_packet.At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.Outputs(); - ASSERT_EQ(1, outputs.NumEntries()); - const std::vector& packets = outputs.Index(0).packets; - ASSERT_EQ(1, packets.size()); - const ImageFrame& output_frame = packets[0].Get(); - cv::Mat diff; - cv::absdiff(input_mat, formats::MatView(&output_frame), diff); - double max_val; - cv::minMaxLoc(diff, nullptr, &max_val); - // Expects that the maximum absolute pixel-by-pixel difference is less - // than 10. - EXPECT_LE(max_val, 10); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc b/mediapipe/calculators/image/opencv_image_encoder_calculator.cc deleted file mode 100644 index 93ec9435f..000000000 --- a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { - -// Calculator to encode raw image frames. This will result in considerable space -// savings if the frames need to be stored on disk. -// -// Example config: -// node { -// calculator: "OpenCvImageEncoderCalculator" -// input_stream: "image" -// output_stream: "encoded_image" -// node_options { -// [type.googleapis.com/mediapipe.OpenCvImageEncoderCalculatorOptions]: { -// quality: 80 -// } -// } -// } -class OpenCvImageEncoderCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - int encoding_quality_; -}; - -absl::Status OpenCvImageEncoderCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); -} - -absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) { - auto options = cc->Options(); - encoding_quality_ = options.quality(); - return absl::OkStatus(); -} - -absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) { - const ImageFrame& image_frame = cc->Inputs().Index(0).Get(); - CHECK_EQ(1, image_frame.ByteDepth()); - - std::unique_ptr encoded_result = - absl::make_unique(); - encoded_result->set_width(image_frame.Width()); - encoded_result->set_height(image_frame.Height()); - - cv::Mat original_mat = formats::MatView(&image_frame); - cv::Mat input_mat; - switch (original_mat.channels()) { - case 1: - input_mat = original_mat; - encoded_result->set_colorspace( - OpenCvImageEncoderCalculatorResults::GRAYSCALE); - break; - case 3: - // OpenCV assumes the image to be BGR order. To use imencode(), do color - // conversion first. - cv::cvtColor(original_mat, input_mat, cv::COLOR_RGB2BGR); - encoded_result->set_colorspace(OpenCvImageEncoderCalculatorResults::RGB); - break; - case 4: - return mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC) - << "4-channel image isn't supported yet"; - default: - return mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported number of channels: " << original_mat.channels(); - } - - std::vector parameters; - parameters.push_back(cv::IMWRITE_JPEG_QUALITY); - parameters.push_back(encoding_quality_); - - std::vector encode_buffer; - // Note that imencode() will store the data in RGB order. - // Check its JpegEncoder::write() in "imgcodecs/src/grfmt_jpeg.cpp" for more - // info. - if (!cv::imencode(".jpg", input_mat, encode_buffer, parameters)) { - return mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC) - << "Fail to encode the image to be jpeg format."; - } - - encoded_result->set_encoded_image(&encode_buffer[0], encode_buffer.size()); - - cc->Outputs().Index(0).Add(encoded_result.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -absl::Status OpenCvImageEncoderCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(OpenCvImageEncoderCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/opencv_image_encoder_calculator.proto b/mediapipe/calculators/image/opencv_image_encoder_calculator.proto deleted file mode 100644 index 0564fb270..000000000 --- a/mediapipe/calculators/image/opencv_image_encoder_calculator.proto +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenCvImageEncoderCalculatorOptions { - extend CalculatorOptions { - optional OpenCvImageEncoderCalculatorOptions ext = 227563646; - } - - // Quality of the encoding. An integer between (0, 100]. - optional int32 quality = 1; -} - -// TODO: Consider renaming it to EncodedImage. -message OpenCvImageEncoderCalculatorResults { - // Pixel data encoded as JPEG. - optional bytes encoded_image = 1; - - // Height of the image data under #1 once decoded. - optional int32 height = 2; - - // Width of the image data under #1 once decoded. - optional int32 width = 3; - - enum ColorSpace { - UNKNOWN = 0; - GRAYSCALE = 1; - RGB = 2; - } - - // Color space used. - optional ColorSpace colorspace = 4; -} diff --git a/mediapipe/calculators/image/opencv_image_encoder_calculator_test.cc b/mediapipe/calculators/image/opencv_image_encoder_calculator_test.cc deleted file mode 100644 index 9d2986579..000000000 --- a/mediapipe/calculators/image/opencv_image_encoder_calculator_test.cc +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -TEST(OpenCvImageEncoderCalculatorTest, TestJpegWithQualities) { - cv::Mat input_mat; - cv::cvtColor(cv::imread(file::JoinPath("./", - "/mediapipe/calculators/" - "image/testdata/dino.jpg")), - input_mat, cv::COLOR_BGR2RGB); - Packet input_packet = MakePacket( - ImageFormat::SRGB, input_mat.size().width, input_mat.size().height); - input_mat.copyTo(formats::MatView(&(input_packet.Get()))); - - std::vector qualities = {50, 80}; - for (int quality : qualities) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie( - absl::Substitute(R"( - calculator: "OpenCvImageEncoderCalculator" - input_stream: "image_frames" - output_stream: "encoded_images" - node_options { - [type.googleapis.com/mediapipe.OpenCvImageEncoderCalculatorOptions]: { - quality: $0 - } - })", - quality)); - CalculatorRunner runner(node_config); - runner.MutableInputs()->Index(0).packets.push_back( - input_packet.At(Timestamp(0))); - MP_ASSERT_OK(runner.Run()); - const auto& outputs = runner.Outputs(); - ASSERT_EQ(1, outputs.NumEntries()); - const std::vector& packets = outputs.Index(0).packets; - ASSERT_EQ(1, packets.size()); - const auto& result = packets[0].Get(); - ASSERT_EQ(input_mat.size().height, result.height()); - ASSERT_EQ(input_mat.size().width, result.width()); - ASSERT_EQ(OpenCvImageEncoderCalculatorResults::RGB, result.colorspace()); - - cv::Mat expected_output = cv::imread( - file::JoinPath("./", absl::Substitute("/mediapipe/calculators/image/" - "testdata/dino_quality_$0.jpg", - quality))); - const std::vector contents_vector(result.encoded_image().begin(), - result.encoded_image().end()); - cv::Mat decoded_output = - cv::imdecode(contents_vector, -1 /* return the loaded image as-is */); - cv::Mat diff; - cv::absdiff(expected_output, decoded_output, diff); - double max_val; - cv::minMaxLoc(diff, nullptr, &max_val); - // Expects that the maximum absolute pixel-by-pixel difference is less - // than 10. - EXPECT_LE(max_val, 10); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/image/opencv_put_text_calculator.cc b/mediapipe/calculators/image/opencv_put_text_calculator.cc deleted file mode 100644 index 82a4b3a53..000000000 --- a/mediapipe/calculators/image/opencv_put_text_calculator.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { - -// Takes in a std::string, draws the text std::string by cv::putText(), and -// outputs an ImageFrame. -// -// Example config: -// node { -// calculator: "OpenCvPutTextCalculator" -// input_stream: "text_to_put" -// output_stream: "out_image_frames" -// } -// TODO: Generalize the calculator for other text use cases. -class OpenCvPutTextCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Process(CalculatorContext* cc) override; -}; - -absl::Status OpenCvPutTextCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); -} - -absl::Status OpenCvPutTextCalculator::Process(CalculatorContext* cc) { - const std::string& text_content = cc->Inputs().Index(0).Get(); - cv::Mat mat = cv::Mat::zeros(640, 640, CV_8UC4); - cv::putText(mat, text_content, cv::Point(15, 70), cv::FONT_HERSHEY_PLAIN, 3, - cv::Scalar(255, 255, 0, 255), 4); - std::unique_ptr output_frame = absl::make_unique( - ImageFormat::SRGBA, mat.size().width, mat.size().height); - mat.copyTo(formats::MatView(output_frame.get())); - cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(OpenCvPutTextCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/recolor_calculator.cc b/mediapipe/calculators/image/recolor_calculator.cc deleted file mode 100644 index 062fb2cb3..000000000 --- a/mediapipe/calculators/image/recolor_calculator.cc +++ /dev/null @@ -1,506 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/image/recolor_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/color.pb.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace { -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kMaskCpuTag[] = "MASK"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kMaskGpuTag[] = "MASK_GPU"; - -inline cv::Vec3b Blend(const cv::Vec3b& color1, const cv::Vec3b& color2, - float weight, int invert_mask, - int adjust_with_luminance) { - weight = (1 - invert_mask) * weight + invert_mask * (1.0f - weight); - - float luminance = - (1 - adjust_with_luminance) * 1.0f + - adjust_with_luminance * - (color1[0] * 0.299 + color1[1] * 0.587 + color1[2] * 0.114) / 255; - - float mix_value = weight * luminance; - - return color1 * (1.0 - mix_value) + color2 * mix_value; -} - -} // namespace - -namespace mediapipe { - -// A calculator to recolor a masked area of an image to a specified color. -// -// A mask image is used to specify where to overlay a user defined color. -// -// Inputs: -// One of the following IMAGE tags: -// IMAGE: An ImageFrame input image in ImageFormat::SRGB. -// IMAGE_GPU: A GpuBuffer input image, RGBA. -// One of the following MASK tags: -// MASK: An ImageFrame input mask in ImageFormat::GRAY8, SRGB, SRGBA, or -// VEC32F1 -// MASK_GPU: A GpuBuffer input mask, RGBA. -// Output: -// One of the following IMAGE tags: -// IMAGE: An ImageFrame output image. -// IMAGE_GPU: A GpuBuffer output image. -// -// Options: -// color_rgb (required): A map of RGB values [0-255]. -// mask_channel (optional): Which channel of mask image is used [RED or ALPHA] -// -// Usage example: -// node { -// calculator: "RecolorCalculator" -// input_stream: "IMAGE_GPU:input_image" -// input_stream: "MASK_GPU:input_mask" -// output_stream: "IMAGE_GPU:output_image" -// node_options: { -// [mediapipe.RecolorCalculatorOptions] { -// color { r: 0 g: 0 b: 255 } -// mask_channel: RED -// } -// } -// } -// -// Note: Cannot mix-match CPU & GPU inputs/outputs. -// CPU-in & CPU-out GPU-in & GPU-out -class RecolorCalculator : public CalculatorBase { - public: - RecolorCalculator() = default; - ~RecolorCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status InitGpu(CalculatorContext* cc); - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - void GlRender(); - - bool initialized_ = false; - std::vector color_; - mediapipe::RecolorCalculatorOptions::MaskChannel mask_channel_; - - bool use_gpu_ = false; - bool invert_mask_ = false; - bool adjust_with_luminance_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint program_ = 0; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(RecolorCalculator); - -// static -absl::Status RecolorCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - bool use_gpu = false; - -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - cc->Inputs().Tag(kGpuBufferTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } - -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kMaskGpuTag)) { - cc->Inputs().Tag(kMaskGpuTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kMaskCpuTag)) { - cc->Inputs().Tag(kMaskCpuTag).Set(); - } - -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kGpuBufferTag)) { - cc->Outputs().Tag(kGpuBufferTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kImageFrameTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); - } - - // Confirm only one of the input streams is present. - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^ - cc->Inputs().HasTag(kGpuBufferTag)); - // Confirm only one of the output streams is present. - RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) ^ - cc->Outputs().HasTag(kGpuBufferTag)); - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kGpuBufferTag)) { - use_gpu_ = true; -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::Process(CalculatorContext* cc) { - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, &cc]() -> absl::Status { - if (!initialized_) { - MP_RETURN_IF_ERROR(InitGpu(cc)); - initialized_ = true; - } - MP_RETURN_IF_ERROR(RenderGpu(cc)); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(RenderCpu(cc)); - } - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - program_ = 0; - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::RenderCpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kMaskCpuTag).IsEmpty()) { - cc->Outputs() - .Tag(kImageFrameTag) - .AddPacket(cc->Inputs().Tag(kImageFrameTag).Value()); - return absl::OkStatus(); - } - // Get inputs and setup output. - const auto& input_img = cc->Inputs().Tag(kImageFrameTag).Get(); - const auto& mask_img = cc->Inputs().Tag(kMaskCpuTag).Get(); - - cv::Mat input_mat = formats::MatView(&input_img); - cv::Mat mask_mat = formats::MatView(&mask_img); - - RET_CHECK(input_mat.channels() == 3); // RGB only. - - if (mask_mat.channels() > 1) { - std::vector channels; - cv::split(mask_mat, channels); - if (mask_channel_ == mediapipe::RecolorCalculatorOptions_MaskChannel_ALPHA) - mask_mat = channels[3]; - else - mask_mat = channels[0]; - } - cv::Mat mask_full; - cv::resize(mask_mat, mask_full, input_mat.size()); - const cv::Vec3b recolor = {color_[0], color_[1], color_[2]}; - - auto output_img = absl::make_unique( - input_img.Format(), input_mat.cols, input_mat.rows); - cv::Mat output_mat = mediapipe::formats::MatView(output_img.get()); - - const int invert_mask = invert_mask_ ? 1 : 0; - const int adjust_with_luminance = adjust_with_luminance_ ? 1 : 0; - - // From GPU shader: - /* - vec4 weight = texture2D(mask, sample_coordinate); - vec4 color1 = texture2D(frame, sample_coordinate); - vec4 color2 = vec4(recolor, 1.0); - - float luminance = dot(color1.rgb, vec3(0.299, 0.587, 0.114)); - float mix_value = weight.MASK_COMPONENT * luminance; - - fragColor = mix(color1, color2, mix_value); - */ - if (mask_img.Format() == ImageFormat::VEC32F1) { - for (int i = 0; i < output_mat.rows; ++i) { - for (int j = 0; j < output_mat.cols; ++j) { - const float weight = mask_full.at(i, j); - output_mat.at(i, j) = - Blend(input_mat.at(i, j), recolor, weight, invert_mask, - adjust_with_luminance); - } - } - } else { - for (int i = 0; i < output_mat.rows; ++i) { - for (int j = 0; j < output_mat.cols; ++j) { - const float weight = mask_full.at(i, j) * (1.0 / 255.0); - output_mat.at(i, j) = - Blend(input_mat.at(i, j), recolor, weight, invert_mask, - adjust_with_luminance); - } - } - } - - cc->Outputs() - .Tag(kImageFrameTag) - .Add(output_img.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::RenderGpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kMaskGpuTag).IsEmpty()) { - cc->Outputs() - .Tag(kGpuBufferTag) - .AddPacket(cc->Inputs().Tag(kGpuBufferTag).Value()); - return absl::OkStatus(); - } -#if !MEDIAPIPE_DISABLE_GPU - // Get inputs and setup output. - const Packet& input_packet = cc->Inputs().Tag(kGpuBufferTag).Value(); - const Packet& mask_packet = cc->Inputs().Tag(kMaskGpuTag).Value(); - - const auto& input_buffer = input_packet.Get(); - const auto& mask_buffer = mask_packet.Get(); - - auto img_tex = gpu_helper_.CreateSourceTexture(input_buffer); - auto mask_tex = gpu_helper_.CreateSourceTexture(mask_buffer); - auto dst_tex = - gpu_helper_.CreateDestinationTexture(img_tex.width(), img_tex.height()); - - // Run recolor shader on GPU. - { - gpu_helper_.BindFramebuffer(dst_tex); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(img_tex.target(), img_tex.name()); - glActiveTexture(GL_TEXTURE2); - glBindTexture(mask_tex.target(), mask_tex.name()); - - GlRender(); - - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } - - // Send result image in GPU packet. - auto output = dst_tex.GetFrame(); - cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp()); - - // Cleanup - img_tex.Release(); - mask_tex.Release(); - dst_tex.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void RecolorCalculator::GlRender() { -#if !MEDIAPIPE_DISABLE_GPU - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(program_); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status RecolorCalculator::LoadOptions(CalculatorContext* cc) { - const auto& options = cc->Options(); - - mask_channel_ = options.mask_channel(); - - if (!options.has_color()) RET_CHECK_FAIL() << "Missing color option."; - - color_.push_back(options.color().r()); - color_.push_back(options.color().g()); - color_.push_back(options.color().b()); - - invert_mask_ = options.invert_mask(); - adjust_with_luminance_ = options.adjust_with_luminance(); - - return absl::OkStatus(); -} - -absl::Status RecolorCalculator::InitGpu(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - std::string mask_component; - switch (mask_channel_) { - case mediapipe::RecolorCalculatorOptions_MaskChannel_UNKNOWN: - case mediapipe::RecolorCalculatorOptions_MaskChannel_RED: - mask_component = "r"; - break; - case mediapipe::RecolorCalculatorOptions_MaskChannel_ALPHA: - mask_component = "a"; - break; - } - - // A shader to blend a color onto an image where the mask > 0. - // The blending is based on the input image luminosity. - const std::string frag_src = R"( - #if __VERSION__ < 130 - #define in varying - #endif // __VERSION__ < 130 - - #ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; - #else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; - #endif // defined(GL_ES) - - #define MASK_COMPONENT )" + mask_component + - R"( - - in vec2 sample_coordinate; - uniform sampler2D frame; - uniform sampler2D mask; - uniform vec3 recolor; - uniform float invert_mask; - uniform float adjust_with_luminance; - - void main() { - vec4 weight = texture2D(mask, sample_coordinate); - vec4 color1 = texture2D(frame, sample_coordinate); - vec4 color2 = vec4(recolor, 1.0); - - weight = mix(weight, 1.0 - weight, invert_mask); - - float luminance = mix(1.0, - dot(color1.rgb, vec3(0.299, 0.587, 0.114)), - adjust_with_luminance); - - float mix_value = weight.MASK_COMPONENT * luminance; - - fragColor = mix(color1, color2, mix_value); - } - )"; - - // shader program and params - mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(), - NUM_ATTRIBUTES, &attr_name[0], attr_location, - &program_); - RET_CHECK(program_) << "Problem initializing the program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "frame"), 1); - glUniform1i(glGetUniformLocation(program_, "mask"), 2); - glUniform3f(glGetUniformLocation(program_, "recolor"), color_[0] / 255.0, - color_[1] / 255.0, color_[2] / 255.0); - glUniform1f(glGetUniformLocation(program_, "invert_mask"), - invert_mask_ ? 1.0f : 0.0f); - glUniform1f(glGetUniformLocation(program_, "adjust_with_luminance"), - adjust_with_luminance_ ? 1.0f : 0.0f); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/recolor_calculator.proto b/mediapipe/calculators/image/recolor_calculator.proto deleted file mode 100644 index abbf0849d..000000000 --- a/mediapipe/calculators/image/recolor_calculator.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -message RecolorCalculatorOptions { - extend CalculatorOptions { - optional RecolorCalculatorOptions ext = 252527117; - } - - enum MaskChannel { - UNKNOWN = 0; - RED = 1; - ALPHA = 2; - } - - // Selects which channel of the MASK input to use for masking. - optional MaskChannel mask_channel = 1 [default = RED]; - - // Color to blend into input image where mask is > 0. - // The blending is based on the input image luminosity. - optional Color color = 2; - - // Swap the meaning of mask values for foreground/background. - optional bool invert_mask = 3 [default = false]; - - // Whether to use the luminance of the input image to further adjust the - // blending weight, to help preserve image textures. - optional bool adjust_with_luminance = 4 [default = true]; -} diff --git a/mediapipe/calculators/image/scale_image_calculator.cc b/mediapipe/calculators/image/scale_image_calculator.cc deleted file mode 100644 index 575268da5..000000000 --- a/mediapipe/calculators/image/scale_image_calculator.cc +++ /dev/null @@ -1,703 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// This Calculator takes an ImageFrame and scales it appropriately. - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "absl/strings/substitute.h" -#include "libyuv/scale.h" -#include "mediapipe/calculators/image/scale_image_calculator.pb.h" -#include "mediapipe/calculators/image/scale_image_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/formats/yuv_image.h" -#include "mediapipe/framework/port/image_resizer.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/proto_ns.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/image_frame_util.h" - -namespace mediapipe { - -namespace { - -// Given an upscaling algorithm, determine which OpenCV interpolation algorithm -// to use. -absl::Status FindInterpolationAlgorithm( - ScaleImageCalculatorOptions::ScaleAlgorithm upscaling_algorithm, - int* interpolation_algorithm) { - switch (upscaling_algorithm) { - case ScaleImageCalculatorOptions::DEFAULT: - *interpolation_algorithm = cv::INTER_CUBIC; - break; - case ScaleImageCalculatorOptions::LINEAR: - *interpolation_algorithm = cv::INTER_LINEAR; - break; - case ScaleImageCalculatorOptions::CUBIC: - *interpolation_algorithm = cv::INTER_CUBIC; - break; - case ScaleImageCalculatorOptions::AREA: - *interpolation_algorithm = cv::INTER_AREA; - break; - case ScaleImageCalculatorOptions::LANCZOS: - *interpolation_algorithm = cv::INTER_LANCZOS4; - break; - case ScaleImageCalculatorOptions::DEFAULT_WITHOUT_UPSCALE: - *interpolation_algorithm = -1; - break; - default: - RET_CHECK_FAIL() << absl::Substitute("Unknown upscaling algorithm: $0", - upscaling_algorithm); - } - return absl::OkStatus(); -} - -void CropImageFrame(const ImageFrame& original, int col_start, int row_start, - int crop_width, int crop_height, ImageFrame* cropped) { - const uint8* src = original.PixelData(); - uint8* dst = cropped->MutablePixelData(); - - int des_y = 0; - for (int y = row_start; y < row_start + crop_height; ++y) { - const uint8* src_line = src + y * original.WidthStep(); - const uint8* src_pixel = src_line + col_start * - original.NumberOfChannels() * - original.ByteDepth(); - uint8* dst_line = dst + des_y * cropped->WidthStep(); - std::memcpy( - dst_line, src_pixel, - crop_width * cropped->NumberOfChannels() * cropped->ByteDepth()); - ++des_y; - } -} - -} // namespace - -// Crops and scales an ImageFrame or YUVImage according to the options; -// The output can be cropped and scaled ImageFrame with the SRGB format. If the -// input is a YUVImage, the output can be a scaled YUVImage (the scaling is done -// using libyuv). Cropping is not yet supported for a YUVImage to a scaled -// YUVImage conversion. -// -// Example config: -// node { -// calculator: "ScaleImageCalculator" -// input_stream: "raw_frames" -// output_stream: "scaled_frames" -// node_options { -// [type.googleapis.com/mediapipe.ScaleImageCalculatorOptions] { -// target_width: 320 -// target_height: 320 -// preserve_aspect_ratio: true -// output_format: SRGB -// algorithm: DEFAULT -// } -// } -// } -// -// ScaleImageCalculator can also create or update a VideoHeader that is -// provided at Timestamp::PreStream on stream VIDEO_HEADER. -// -// Example config: -// node { -// calculator: "ScaleImageCalculator" -// input_stream: "FRAMES:ycbcr_frames" -// input_stream: "VIDEO_HEADER:ycbcr_frames_header" # Optional. -// output_stream: "FRAMES:srgb_frames" -// output_stream: "VIDEO_HEADER:srgb_frames_header" # Independently Optional. -// node_options { -// [type.googleapis.com/mediapipe.ScaleImageCalculatorOptions] { -// target_width: 320 -// target_height: 320 -// preserve_aspect_ratio: true -// output_format: SRGB -// algorithm: DEFAULT -// } -// } -// } -// -// The calculator options can be overrided with an input stream -// "OVERRIDE_OPTIONS". If this is provided, and non-empty at PreStream, the -// calculator options proto is merged with the proto provided in this packet -// (fields are overwritten in the original options) and the -// initialization happens in Process at PreStream, and not at Open. -class ScaleImageCalculator : public CalculatorBase { - public: - ScaleImageCalculator(); - ~ScaleImageCalculator() override; - - static absl::Status GetContract(CalculatorContract* cc) { - ScaleImageCalculatorOptions options = - cc->Options(); - - CollectionItemId input_data_id = cc->Inputs().GetId("FRAMES", 0); - if (!input_data_id.IsValid()) { - input_data_id = cc->Inputs().GetId("", 0); - } - CollectionItemId output_data_id = cc->Outputs().GetId("FRAMES", 0); - if (!output_data_id.IsValid()) { - output_data_id = cc->Outputs().GetId("", 0); - } - - if (cc->Inputs().HasTag("VIDEO_HEADER")) { - cc->Inputs().Tag("VIDEO_HEADER").Set(); - } - if (options.has_input_format() && - options.input_format() == ImageFormat::YCBCR420P) { - cc->Inputs().Get(input_data_id).Set(); - } else { - cc->Inputs().Get(input_data_id).Set(); - } - - if (cc->Outputs().HasTag("VIDEO_HEADER")) { - cc->Outputs().Tag("VIDEO_HEADER").Set(); - } - if (options.has_output_format() && - options.output_format() == ImageFormat::YCBCR420P) { - RET_CHECK_EQ(ImageFormat::YCBCR420P, options.input_format()); - cc->Outputs().Get(output_data_id).Set(); - } else { - cc->Outputs().Get(output_data_id).Set(); - } - - if (cc->Inputs().HasTag("OVERRIDE_OPTIONS")) { - cc->Inputs().Tag("OVERRIDE_OPTIONS").Set(); - } - return absl::OkStatus(); - } - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Initialize some data members from options_. This can be called either from - // Open or Process depending on whether OVERRIDE_OPTIONS is used. - absl::Status InitializeFromOptions(); - // Initialize crop and output parameters based on set member variable - // values. This function will also send the header information on - // the VIDEO_HEADER stream if it hasn't been done yet. - absl::Status InitializeFrameInfo(CalculatorContext* cc); - // Validate that input_format_ and output_format_ are supported image - // formats. - absl::Status ValidateImageFormats() const; - // Validate that the image frame has the proper format and dimensions. - // If the dimensions and format weren't initialized by the header, - // then the first frame on which this function is called is used - // to initialize. - absl::Status ValidateImageFrame(CalculatorContext* cc, - const ImageFrame& image_frame); - // Validate that the YUV image has the proper dimensions. If the - // dimensions weren't initialized by the header, then the first image - // on which this function is called is used to initialize. - absl::Status ValidateYUVImage(CalculatorContext* cc, - const YUVImage& yuv_image); - - bool has_header_; // True if the input stream has a header. - int input_width_; - int input_height_; - int crop_width_; - int crop_height_; - int col_start_; - int row_start_; - int output_width_; - int output_height_; - ImageFormat::Format input_format_; - ImageFormat::Format output_format_; - int interpolation_algorithm_; - - // The "DATA" input stream. - CollectionItemId input_data_id_; - // The "DATA" output stream. - CollectionItemId output_data_id_; - VideoHeader input_video_header_; - - // Whether the header information was sent on the VIDEO_HEADER stream. - bool header_sent_ = false; - - // The alignment boundary that newly created images should have. - int alignment_boundary_; - - ScaleImageCalculatorOptions options_; - - // Efficient image resizer with gamma correction and optional sharpening. - std::unique_ptr downscaler_; -}; - -REGISTER_CALCULATOR(ScaleImageCalculator); - -ScaleImageCalculator::ScaleImageCalculator() {} - -ScaleImageCalculator::~ScaleImageCalculator() {} - -absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) { - MP_RETURN_IF_ERROR( - scale_image::FindCropDimensions(input_width_, input_height_, // - options_.min_aspect_ratio(), // - options_.max_aspect_ratio(), // - &crop_width_, &crop_height_, // - &col_start_, &row_start_)); - MP_RETURN_IF_ERROR( - scale_image::FindOutputDimensions(crop_width_, crop_height_, // - options_.target_width(), // - options_.target_height(), // - options_.preserve_aspect_ratio(), // - options_.scale_to_multiple_of(), // - &output_width_, &output_height_)); - MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(), - &interpolation_algorithm_)); - if (interpolation_algorithm_ == -1 && - (output_width_ > crop_width_ || output_height_ > crop_height_)) { - output_width_ = crop_width_; - output_height_ = crop_height_; - } - VLOG(1) << "Image scaling parameters:" - << "\ninput_width_ " << input_width_ // - << "\ninput_height_ " << input_height_ // - << "\ninput_format_ " << input_format_ // - << "\ncrop_width_ " << crop_width_ // - << "\ncrop_height_ " << crop_height_ // - << "\ncol_start_ " << col_start_ // - << "\nrow_start_ " << row_start_ // - << "\noutput_width_ " << output_width_ // - << "\noutput_height_ " << output_height_ // - << "\noutput_format_ " << output_format_ // - << "\nOpenCV interpolation algorithm " << interpolation_algorithm_; - if (!header_sent_ && cc->Outputs().UsesTags() && - cc->Outputs().HasTag("VIDEO_HEADER")) { - header_sent_ = true; - auto header = absl::make_unique(); - *header = input_video_header_; - header->width = output_width_; - header->height = output_height_; - header->format = output_format_; - LOG(INFO) << "OUTPUTTING HEADER on stream"; - cc->Outputs() - .Tag("VIDEO_HEADER") - .Add(header.release(), Timestamp::PreStream()); - cc->Outputs().Tag("VIDEO_HEADER").Close(); - } - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - - input_data_id_ = cc->Inputs().GetId("FRAMES", 0); - if (!input_data_id_.IsValid()) { - input_data_id_ = cc->Inputs().GetId("", 0); - } - output_data_id_ = cc->Outputs().GetId("FRAMES", 0); - if (!output_data_id_.IsValid()) { - output_data_id_ = cc->Outputs().GetId("", 0); - } - - // The output packets are at the same timestamp as the input. - cc->Outputs().Get(output_data_id_).SetOffset(mediapipe::TimestampDiff(0)); - - has_header_ = false; - input_width_ = 0; - input_height_ = 0; - crop_width_ = 0; - crop_height_ = 0; - output_width_ = 0; - output_height_ = 0; - bool has_override_options = cc->Inputs().HasTag("OVERRIDE_OPTIONS"); - - if (!has_override_options) { - MP_RETURN_IF_ERROR(InitializeFromOptions()); - } - - if (!cc->Inputs().Get(input_data_id_).Header().IsEmpty()) { - // If the input stream has a header then our output stream also has a - // header. - - if (has_override_options) { - // It's not possible to use OVERRIDE_OPTIONS when the main input stream - // has a header. At this point in the code, the ScaleImageCalculator - // config may be changed by the new options at PreStream, so the output - // header can't be determined. - return absl::InvalidArgumentError( - "OVERRIDE_OPTIONS stream can't be used when the main input stream " - "has a header."); - } - input_video_header_ = - cc->Inputs().Get(input_data_id_).Header().Get(); - - input_format_ = input_video_header_.format; - if (options_.has_input_format()) { - RET_CHECK_EQ(input_format_, options_.input_format()) - << "The input header format does not match the input_format option."; - } - - input_width_ = input_video_header_.width; - input_height_ = input_video_header_.height; - - if (options_.has_output_format()) { - output_format_ = options_.output_format(); - } else { - output_format_ = input_format_; - } - - const bool is_positive_and_even = - (options_.scale_to_multiple_of() >= 1) && - (options_.scale_to_multiple_of() % 2 == 0); - - if (output_format_ == ImageFormat::YCBCR420P) { - RET_CHECK(is_positive_and_even) - << "ScaleImageCalculator always outputs width and height that are " - "divisible by 2 when output format is YCbCr420P. To scale to " - "width and height of odd numbers, the output format must be SRGB."; - } else if (options_.preserve_aspect_ratio()) { - RET_CHECK(options_.scale_to_multiple_of() == 2) - << "ScaleImageCalculator always outputs width and height that are " - "divisible by 2 when preserving aspect ratio. If you'd like to " - "set scale_to_multiple_of to something other than 2, please " - "set preserve_aspect_ratio to false."; - } - - if (input_width_ > 0 && input_height_ > 0 && - input_format_ != ImageFormat::UNKNOWN && - output_format_ != ImageFormat::UNKNOWN) { - MP_RETURN_IF_ERROR(ValidateImageFormats()); - MP_RETURN_IF_ERROR(InitializeFrameInfo(cc)); - std::unique_ptr output_header(new VideoHeader()); - *output_header = input_video_header_; - output_header->format = output_format_; - output_header->width = output_width_; - output_header->height = output_height_; - cc->Outputs() - .Get(output_data_id_) - .SetHeader(Adopt(output_header.release())); - has_header_ = true; - } else { - LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient " - "information. " - "Dropping VideoHeader and trying to deduce needed " - "information."; - input_width_ = 0; - input_height_ = 0; - if (!options_.has_input_format()) { - input_format_ = ImageFormat::UNKNOWN; - } - output_format_ = ImageFormat::UNKNOWN; - } - } - - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::InitializeFromOptions() { - if (options_.has_input_format()) { - input_format_ = options_.input_format(); - } else { - input_format_ = ImageFormat::UNKNOWN; - } - - alignment_boundary_ = 16; - if (options_.alignment_boundary() > 0) { - alignment_boundary_ = options_.alignment_boundary(); - } - - downscaler_.reset(new ImageResizer(options_.post_sharpening_coefficient())); - - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::ValidateImageFormats() const { - RET_CHECK_NE(input_format_, ImageFormat::UNKNOWN) - << "The input image format was UNKNOWN."; - RET_CHECK_NE(output_format_, ImageFormat::UNKNOWN) - << "The output image format was set to UNKNOWN."; - // TODO Remove these conditions. - RET_CHECK(output_format_ == ImageFormat::SRGB || - (input_format_ == output_format_ && - output_format_ == ImageFormat::YCBCR420P)) - << "Outputting YCbCr420P images from SRGB input is not yet supported"; - RET_CHECK(input_format_ == output_format_ || - input_format_ == ImageFormat::YCBCR420P) - << "Conversion of the color space (except from " - "YCbCr420P to SRGB) is not yet supported."; - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::ValidateImageFrame( - CalculatorContext* cc, const ImageFrame& image_frame) { - if (!has_header_) { - if (input_width_ != image_frame.Width() || - input_height_ != image_frame.Height() || - input_format_ != image_frame.Format()) { - // Set the dimensions based on the image frame. There was no header. - input_width_ = image_frame.Width(); - input_height_ = image_frame.Height(); - RET_CHECK(input_width_ > 0 && input_height_ > 0) << absl::StrCat( - "The input image did not have positive dimensions. dimensions: ", - input_width_, "x", input_height_); - input_format_ = image_frame.Format(); - if (options_.has_input_format()) { - RET_CHECK_EQ(input_format_, options_.input_format()) - << "The input image format does not match the input_format option."; - } - if (options_.has_output_format()) { - output_format_ = options_.output_format(); - } else { - output_format_ = input_format_; - } - MP_RETURN_IF_ERROR(InitializeFrameInfo(cc)); - } - MP_RETURN_IF_ERROR(ValidateImageFormats()); - } else { - if (input_width_ != image_frame.Width() || - input_height_ != image_frame.Height()) { - return tool::StatusFail(absl::StrCat( - "If a header specifies a width and a height, then image frames on " - "the stream must have that size. Received frame of size ", - image_frame.Width(), "x", image_frame.Height(), " but expected ", - input_width_, "x", input_height_)); - } - if (input_format_ != image_frame.Format()) { - std::string image_frame_format_desc, input_format_desc; -#ifdef MEDIAPIPE_MOBILE - image_frame_format_desc = std::to_string(image_frame.Format()); - input_format_desc = std::to_string(input_format_); -#else - const proto_ns::EnumDescriptor* desc = ImageFormat::Format_descriptor(); - image_frame_format_desc = - desc->FindValueByNumber(image_frame.Format())->DebugString(); - input_format_desc = desc->FindValueByNumber(input_format_)->DebugString(); -#endif // MEDIAPIPE_MOBILE - return tool::StatusFail(absl::StrCat( - "If a header specifies a format, then image frames on " - "the stream must have that format. Actual format ", - image_frame_format_desc, " but expected ", input_format_desc)); - } - } - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc, - const YUVImage& yuv_image) { - CHECK_EQ(input_format_, ImageFormat::YCBCR420P); - if (!has_header_) { - if (input_width_ != yuv_image.width() || - input_height_ != yuv_image.height()) { - // Set the dimensions based on the YUV image. There was no header. - input_width_ = yuv_image.width(); - input_height_ = yuv_image.height(); - RET_CHECK(input_width_ > 0 && input_height_ > 0) << absl::StrCat( - "The input image did not have positive dimensions. dimensions: ", - input_width_, "x", input_height_); - if (options_.has_output_format()) { - output_format_ = options_.output_format(); - } else { - output_format_ = input_format_; - } - MP_RETURN_IF_ERROR(InitializeFrameInfo(cc)); - } - MP_RETURN_IF_ERROR(ValidateImageFormats()); - } else { - if (input_width_ != yuv_image.width() || - input_height_ != yuv_image.height()) { - return tool::StatusFail(absl::StrCat( - "If a header specifies a width and a height, then YUV images on " - "the stream must have that size. Additionally, all YUV images in " - "a stream must have the same size. Received frame of size ", - yuv_image.width(), "x", yuv_image.height(), " but expected ", - input_width_, "x", input_height_)); - } - } - return absl::OkStatus(); -} - -absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) { - if (cc->InputTimestamp() == Timestamp::PreStream()) { - if (cc->Inputs().HasTag("OVERRIDE_OPTIONS")) { - if (cc->Inputs().Tag("OVERRIDE_OPTIONS").IsEmpty()) { - return absl::InvalidArgumentError( - "The OVERRIDE_OPTIONS input stream must be non-empty at PreStream " - "time if used."); - } - options_.MergeFrom(cc->Inputs() - .Tag("OVERRIDE_OPTIONS") - .Get()); - MP_RETURN_IF_ERROR(InitializeFromOptions()); - } - if (cc->Inputs().UsesTags() && cc->Inputs().HasTag("VIDEO_HEADER") && - !cc->Inputs().Tag("VIDEO_HEADER").IsEmpty()) { - input_video_header_ = cc->Inputs().Tag("VIDEO_HEADER").Get(); - } - if (cc->Inputs().Get(input_data_id_).IsEmpty()) { - return absl::OkStatus(); - } - } - - cc->GetCounter("Inputs")->Increment(); - const ImageFrame* image_frame; - ImageFrame converted_image_frame; - if (input_format_ == ImageFormat::YCBCR420P) { - const YUVImage* yuv_image = - &cc->Inputs().Get(input_data_id_).Get(); - MP_RETURN_IF_ERROR(ValidateYUVImage(cc, *yuv_image)); - - if (output_format_ == ImageFormat::SRGB) { - // TODO: For ease of implementation, YUVImage is converted to - // ImageFrame immediately, before cropping and scaling. Investigate how to - // make color space conversion more efficient when cropping or scaling is - // also needed. - image_frame_util::YUVImageToImageFrame(*yuv_image, &converted_image_frame, - options_.use_bt709()); - image_frame = &converted_image_frame; - } else if (output_format_ == ImageFormat::YCBCR420P) { - RET_CHECK(row_start_ == 0 && col_start_ == 0 && - crop_width_ == input_width_ && crop_height_ == input_height_) - << "ScaleImageCalculator only supports scaling on YUVImages. To crop " - "images, the output format must be SRGB."; - - // Scale the YUVImage and output without converting the color space. - const int y_size = output_width_ * output_height_; - const int uv_size = output_width_ * output_height_ / 4; - std::unique_ptr yuv_data(new uint8_t[y_size + uv_size * 2]); - uint8* y = yuv_data.get(); - uint8* u = y + y_size; - uint8* v = u + uv_size; - RET_CHECK_EQ(0, I420Scale(yuv_image->data(0), yuv_image->stride(0), - yuv_image->data(1), yuv_image->stride(1), - yuv_image->data(2), yuv_image->stride(2), - yuv_image->width(), yuv_image->height(), y, - output_width_, u, output_width_ / 2, v, - output_width_ / 2, output_width_, - output_height_, libyuv::kFilterBox)); - auto output_image = absl::make_unique( - libyuv::FOURCC_I420, std::move(yuv_data), y, output_width_, u, - output_width_ / 2, v, output_width_ / 2, output_width_, - output_height_); - cc->GetCounter("Outputs Scaled")->Increment(); - if (yuv_image->width() >= output_width_ && - yuv_image->height() >= output_height_) { - cc->GetCounter("Downscales")->Increment(); - } else if (interpolation_algorithm_ != -1) { - cc->GetCounter("Upscales")->Increment(); - } - cc->Outputs() - .Get(output_data_id_) - .Add(output_image.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - } else { - image_frame = &cc->Inputs().Get(input_data_id_).Get(); - MP_RETURN_IF_ERROR(ValidateImageFrame(cc, *image_frame)); - } - - std::unique_ptr cropped_image; - if (crop_width_ < input_width_ || crop_height_ < input_height_) { - cc->GetCounter("Crops")->Increment(); - // TODO Do the crop as a range restrict inside OpenCV code below. - cropped_image.reset(new ImageFrame(image_frame->Format(), crop_width_, - crop_height_, alignment_boundary_)); - if (image_frame->ByteDepth() == 1 || image_frame->ByteDepth() == 2) { - CropImageFrame(*image_frame, col_start_, row_start_, crop_width_, - crop_height_, cropped_image.get()); - } else { - return tool::StatusInvalid( - "Input format does not have ByteDepth of 1 or 2."); - } - - // Update the image_frame to point to the cropped image. The - // unique_ptr will take care of deleting the cropped image when the - // function returns. - image_frame = cropped_image.get(); - } - - // Skip later operations if no scaling is necessary. - if (crop_width_ == output_width_ && crop_height_ == output_height_) { - // Efficiently use either the cropped image or the original image. - if (image_frame == cropped_image.get()) { - if (options_.set_alignment_padding()) { - cropped_image->SetAlignmentPaddingAreas(); - } - cc->GetCounter("Outputs Cropped")->Increment(); - cc->Outputs() - .Get(output_data_id_) - .Add(cropped_image.release(), cc->InputTimestamp()); - } else { - if (options_.alignment_boundary() <= 0 && - (!options_.set_alignment_padding() || image_frame->IsContiguous())) { - // Any alignment is acceptable and we don't need to clear the - // alignment padding (either because the user didn't request it - // or because the data is contiguous). - cc->GetCounter("Outputs Inputs")->Increment(); - cc->Outputs() - .Get(output_data_id_) - .AddPacket(cc->Inputs().Get(input_data_id_).Value()); - } else { - // Make a copy with the correct alignment. - std::unique_ptr output_frame(new ImageFrame()); - output_frame->CopyFrom(*image_frame, alignment_boundary_); - if (options_.set_alignment_padding()) { - output_frame->SetAlignmentPaddingAreas(); - } - cc->GetCounter("Outputs Aligned")->Increment(); - cc->Outputs() - .Get(output_data_id_) - .Add(output_frame.release(), cc->InputTimestamp()); - } - } - return absl::OkStatus(); - } - - // Rescale the image frame. - std::unique_ptr output_frame(new ImageFrame()); - if (image_frame->Width() >= output_width_ && - image_frame->Height() >= output_height_) { - // Downscale. - cc->GetCounter("Downscales")->Increment(); - cv::Mat input_mat = ::mediapipe::formats::MatView(image_frame); - output_frame->Reset(image_frame->Format(), output_width_, output_height_, - alignment_boundary_); - cv::Mat output_mat = ::mediapipe::formats::MatView(output_frame.get()); - downscaler_->Resize(input_mat, &output_mat); - } else { - // Upscale. If upscaling is disallowed, output_width_ and output_height_ are - // the same as the input/crop width and height. - image_frame_util::RescaleImageFrame( - *image_frame, output_width_, output_height_, alignment_boundary_, - interpolation_algorithm_, output_frame.get()); - if (interpolation_algorithm_ != -1) { - cc->GetCounter("Upscales")->Increment(); - } - } - - if (options_.set_alignment_padding()) { - cc->GetCounter("Pads")->Increment(); - output_frame->SetAlignmentPaddingAreas(); - } - - cc->GetCounter("Outputs Scaled")->Increment(); - cc->Outputs() - .Get(output_data_id_) - .Add(output_frame.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/scale_image_calculator.proto b/mediapipe/calculators/image/scale_image_calculator.proto deleted file mode 100644 index e51ccafaa..000000000 --- a/mediapipe/calculators/image/scale_image_calculator.proto +++ /dev/null @@ -1,114 +0,0 @@ -// Options for ScaleImageCalculator. -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/framework/formats/image_format.proto"; - -// Order of operations. -// 1) Crop the image to fit within min_aspect_ratio and max_aspect_ratio. -// 2) Scale and convert the image to fit inside target_width x target_height -// using the specified scaling algorithm. (maintaining the aspect -// ratio if preserve_aspect_ratio is true). -// The output width and height will be divisible by 2, by default. It is -// possible to output width and height that are odd numbers when the output -// format is SRGB and the aspect ratio is left unpreserved. See -// scale_to_multiple_of for details. -message ScaleImageCalculatorOptions { - extend CalculatorOptions { - optional ScaleImageCalculatorOptions ext = 66237115; - } - - // Target output width and height. The final output's size may vary - // depending on the other options below. If unset, use the same width - // or height as the input. If only one is set then determine the other - // from the aspect ratio (after cropping). The output width and height - // will be divisible by 2, by default. - optional int32 target_width = 1; - optional int32 target_height = 2; - - // If true, the image is scaled up or down proportionally so that it - // fits inside the box represented by target_width and target_height. - // Otherwise it is scaled to fit target_width and target_height - // completely. In any case, the aspect ratio that is preserved is - // that after cropping to the minimum/maximum aspect ratio. Additionally, if - // true, the output width and height will be divisible by 2. - optional bool preserve_aspect_ratio = 3 [default = true]; - - // If ratio is positive, crop the image to this minimum and maximum - // aspect ratio (preserving the center of the frame). This is done - // before scaling. The string must contain "/", so to disable cropping, - // set both to "0/1". - // For example, for a min_aspect_ratio of "9/16" and max of "16/9" the - // following cropping will occur: - // 1920x1080 (which is 16:9) is not cropped - // 640x1024 (which is 10:16) is not cropped - // 640x320 (which is 2:1) cropped to 568x320 (just under 16/9) - // 96x480 (which is 1:5), cropped to 96x170 (just over 9/16) - // The resultant frame will always be between (or at) the - // min_aspect_ratio and max_aspect_ratio. - optional string min_aspect_ratio = 4 [default = "9/16"]; - optional string max_aspect_ratio = 5 [default = "16/9"]; - - // If unset, use the same format as the input. - // NOTE: in the current implementation, the output format (either specified - // in the output_format option or inherited from the input format) must be - // SRGB. It can be YCBCR420P if the input_format is also the same. - optional ImageFormat.Format output_format = 6; - - enum ScaleAlgorithm { - DEFAULT = 0; - LINEAR = 1; - CUBIC = 2; - AREA = 3; - LANCZOS = 4; - DEFAULT_WITHOUT_UPSCALE = 5; // Option to disallow upscaling. - } - - // The upscaling algorithm to use. The default is to use CUBIC. Note that - // downscaling unconditionally uses DDA; see image_processing:: - // AffineGammaResizer for documentation. - optional ScaleAlgorithm algorithm = 7 [default = DEFAULT]; - - // The output image will have this alignment. If set to zero, then - // any alignment could be used. If set to one, the output image will - // be stored contiguously. - optional int32 alignment_boundary = 8 [default = 16]; - - // Set the alignment padding area to deterministic values (as opposed - // to possibly leaving it as uninitialized memory). The padding is - // the space between the pixel values in a row and the end of the row - // (which may be different due to alignment requirements on the length - // of a row). - optional bool set_alignment_padding = 9 [default = true]; - - optional bool OBSOLETE_skip_linear_rgb_conversion = 10 [default = false]; - - // Applies sharpening for downscaled images as post-processing. See - // image_processing::AffineGammaResizer for documentation. - optional float post_sharpening_coefficient = 11 [default = 0.0]; - - // If input_format is YCBCR420P, input packets contain a YUVImage. If - // input_format is a format other than YCBCR420P or is unset, input packets - // contain an ImageFrame. - // NOTE: in the current implementation, the input format (either specified - // in the input_format option or inferred from the input packets) must be - // SRGB or YCBCR420P. - optional ImageFormat.Format input_format = 12; - - // If set to 2, the target width and height will be rounded-down - // to the nearest even number. If set to any positive value other than 2, - // preserve_aspect_ratio must be false and the target width and height will be - // rounded-down to multiples of the given value. If set to any value less than - // 1, it will be treated like 1. - // NOTE: If set to an odd number, the output format must be SRGB. - optional int32 scale_to_multiple_of = 13 [default = 2]; - - // If true, assume the input YUV is BT.709 (this is the HDTV standard, so most - // content is likely using it). If false use the previous assumption of BT.601 - // (mid-80s standard). Ideally this information should be contained in the - // input YUV Frame, but as of 02/06/2019, it's not. Once this info is baked - // in, this flag becomes useless. - optional bool use_bt709 = 14 [default = false]; -} diff --git a/mediapipe/calculators/image/scale_image_utils.cc b/mediapipe/calculators/image/scale_image_utils.cc deleted file mode 100644 index 738e83da0..000000000 --- a/mediapipe/calculators/image/scale_image_utils.cc +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/scale_image_utils.h" - -#include - -#include - -#include "absl/strings/str_split.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace scale_image { - -namespace { -double ParseRational(const std::string& rational) { - const std::vector v = absl::StrSplit(rational, '/'); - const double numerator = std::strtod(v[0].c_str(), nullptr); - const double denominator = std::strtod(v[1].c_str(), nullptr); - return numerator / denominator; -} -} // namespace - -absl::Status FindCropDimensions(int input_width, int input_height, // - const std::string& min_aspect_ratio, // - const std::string& max_aspect_ratio, // - int* crop_width, int* crop_height, // - int* col_start, int* row_start) { - CHECK(crop_width); - CHECK(crop_height); - CHECK(col_start); - CHECK(row_start); - - double min_aspect_ratio_q = 0.0; - double max_aspect_ratio_q = 0.0; - if (!min_aspect_ratio.empty()) { - min_aspect_ratio_q = ParseRational(min_aspect_ratio); - } - if (!max_aspect_ratio.empty()) { - max_aspect_ratio_q = ParseRational(max_aspect_ratio); - } - - *crop_width = input_width; - *crop_height = input_height; - *col_start = 0; - *row_start = 0; - - // Determine the current aspect ratio. - const double aspect_ratio = - static_cast(input_width) / static_cast(input_height); - - if (!std::isinf(max_aspect_ratio_q) && !std::isinf(min_aspect_ratio_q)) { - if (max_aspect_ratio_q > 0 && aspect_ratio > max_aspect_ratio_q) { - // Determine the width based on the height multiplied by the max - // aspect ratio. - *crop_width = static_cast(static_cast(input_height) * - max_aspect_ratio_q); - *crop_width = (*crop_width / 2) * 2; - // The col_start should be half the difference between the input width - // and the output width. - *col_start = (input_width - *crop_width) / 2; - } else if (min_aspect_ratio_q > 0 && aspect_ratio < min_aspect_ratio_q) { - // Determine the height based on the width divided by the min - // aspect ratio. - *crop_height = static_cast(static_cast(input_width) / - min_aspect_ratio_q); - *crop_height = (*crop_height / 2) * 2; - *row_start = (input_height - *crop_height) / 2; - } - } - - CHECK_LE(*crop_width, input_width); - CHECK_LE(*crop_height, input_height); - return absl::OkStatus(); -} - -absl::Status FindOutputDimensions(int input_width, // - int input_height, // - int target_width, // - int target_height, // - bool preserve_aspect_ratio, // - int scale_to_multiple_of, // - int* output_width, int* output_height) { - CHECK(output_width); - CHECK(output_height); - - if (preserve_aspect_ratio) { - RET_CHECK(scale_to_multiple_of == 2) - << "FindOutputDimensions always outputs width and height that are " - "divisible by 2 when preserving aspect ratio. If you'd like to " - "set scale_to_multiple_of to something other than 2, please " - "set preserve_aspect_ratio to false."; - } - - if (scale_to_multiple_of < 1) scale_to_multiple_of = 1; - - if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) { - if (target_width <= 0) { - target_width = input_width; - } - if (target_height <= 0) { - target_height = input_height; - } - - target_width -= target_width % scale_to_multiple_of; - target_height -= target_height % scale_to_multiple_of; - - *output_width = target_width; - *output_height = target_height; - - return absl::OkStatus(); - } - - if (target_width > 0) { - // Try setting the height based on the width and the aspect ratio. - int try_width = target_width; - int try_height = static_cast(static_cast(target_width) / - static_cast(input_width) * - static_cast(input_height)); - try_width = (try_width / 2) * 2; - try_height = (try_height / 2) * 2; - - if (target_height <= 0 || try_height <= target_height) { - // The resulting height based on the target width and aspect ratio - // was within the image, so use these dimensions. - *output_width = try_width; - *output_height = try_height; - return absl::OkStatus(); - } - } - - if (target_height > 0) { - // Try setting the width based on the height and the aspect ratio. - int try_height = target_height; - int try_width = static_cast(static_cast(target_height) / - static_cast(input_height) * - static_cast(input_width)); - try_width = (try_width / 2) * 2; - try_height = (try_height / 2) * 2; - - if (target_width <= 0 || try_width <= target_width) { - // The resulting width based on the target width and aspect ratio - // was within the image, so use these dimensions. - *output_width = try_width; - *output_height = try_height; - return absl::OkStatus(); - } - } - RET_CHECK_FAIL() - << "Unable to set output dimensions based on target dimensions."; -} - -} // namespace scale_image -} // namespace mediapipe diff --git a/mediapipe/calculators/image/scale_image_utils.h b/mediapipe/calculators/image/scale_image_utils.h deleted file mode 100644 index c2c0b8f7c..000000000 --- a/mediapipe/calculators/image/scale_image_utils.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Utilities for scaling operations defined by ScaleImageCalculatorOptions. -#ifndef MEDIAPIPE_IMAGE_SCALE_IMAGE_UTILS_H_ -#define MEDIAPIPE_IMAGE_SCALE_IMAGE_UTILS_H_ - -#include - -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace scale_image { - -// Given a width and height and min and max aspect ratios, determine the -// target width and height and column and row starts such that the target -// is a centered, cropped portion of the image that falls within the min -// and max aspect ratio. If either the min or max aspect ratio argument -// is empty or has a 0 in the numerator or denominator then it is ignored. -absl::Status FindCropDimensions(int input_width, int input_height, // - const std::string& min_aspect_ratio, // - const std::string& max_aspect_ratio, // - int* crop_width, int* crop_height, // - int* col_start, int* row_start); - -// Given an input width and height, a target width and height, whether to -// preserve the aspect ratio, and whether to round-down to the multiple of a -// given number nearest to the targets, determine the output width and height. -// If target_width or target_height is non-positive, then they will be set to -// the input_width and input_height respectively. If scale_to_multiple_of is -// less than 1, it will be treated like 1. The output_width and -// output_height will be reduced as necessary to preserve_aspect_ratio if the -// option is specified. If preserving the aspect ratio is desired, you must set -// scale_to_multiple_of to 2. -absl::Status FindOutputDimensions(int input_width, int input_height, // - int target_width, - int target_height, // - bool preserve_aspect_ratio, // - int scale_to_multiple_of, // - int* output_width, int* output_height); - -} // namespace scale_image -} // namespace mediapipe - -#endif // MEDIAPIPE_IMAGE_SCALE_IMAGE_UTILS_H_ diff --git a/mediapipe/calculators/image/scale_image_utils_test.cc b/mediapipe/calculators/image/scale_image_utils_test.cc deleted file mode 100644 index 14a58e762..000000000 --- a/mediapipe/calculators/image/scale_image_utils_test.cc +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/image/scale_image_utils.h" - -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace scale_image { -namespace { - -TEST(ScaleImageUtilsTest, FindCropDimensions) { - int crop_width; - int crop_height; - int col_start; - int row_start; - // No cropping because aspect ratios should be ignored. - MP_ASSERT_OK(FindCropDimensions(50, 100, "0/1", "1/0", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(50, crop_width); - EXPECT_EQ(100, crop_height); - EXPECT_EQ(0, row_start); - EXPECT_EQ(0, col_start); - - // Tests proto examples. - // 16:9 aspect ratio, should be unchanged. - MP_ASSERT_OK(FindCropDimensions(1920, 1080, "9/16", "16/9", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(0, col_start); - EXPECT_EQ(1920, crop_width); - EXPECT_EQ(0, row_start); - EXPECT_EQ(1080, crop_height); - // 10:16 aspect ratio, should be unchanged. - MP_ASSERT_OK(FindCropDimensions(640, 1024, "9/16", "16/9", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(0, col_start); - EXPECT_EQ(640, crop_width); - EXPECT_EQ(0, row_start); - EXPECT_EQ(1024, crop_height); - - // 2:1 aspect ratio, width is cropped. - MP_ASSERT_OK(FindCropDimensions(640, 320, "9/16", "16/9", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(36, col_start); - EXPECT_EQ(568, crop_width); - EXPECT_EQ(0, row_start); - EXPECT_EQ(320, crop_height); - // 1:5 aspect ratio, height is cropped. - MP_ASSERT_OK(FindCropDimensions(96, 480, "9/16", "16/9", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(0, col_start); - EXPECT_EQ(96, crop_width); - EXPECT_EQ(155, row_start); - EXPECT_EQ(170, crop_height); - - // Tests min = max, crops width. - MP_ASSERT_OK(FindCropDimensions(200, 100, "1/1", "1/1", &crop_width, - &crop_height, &col_start, &row_start)); - EXPECT_EQ(50, col_start); - EXPECT_EQ(100, crop_width); - EXPECT_EQ(0, row_start); - EXPECT_EQ(100, crop_height); -} - -TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) { - int output_width; - int output_height; - // Not scale. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width, - &output_height)); - EXPECT_EQ(200, output_width); - EXPECT_EQ(100, output_height); - // Not scale with odd input size. - MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width, - &output_height)); - EXPECT_EQ(201, output_width); - EXPECT_EQ(101, output_height); - // Scale down by 1/2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width, - &output_height)); - EXPECT_EQ(100, output_width); - EXPECT_EQ(50, output_height); - // Scale up, doubling dimensions. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width, - &output_height)); - EXPECT_EQ(400, output_width); - EXPECT_EQ(200, output_height); - // Fits a 2:1 image into a 150 x 150 box. Output dimensions are always - // visible by 2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width, - &output_height)); - EXPECT_EQ(150, output_width); - EXPECT_EQ(74, output_height); - // Fits a 2:1 image into a 400 x 50 box. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width, - &output_height)); - EXPECT_EQ(100, output_width); - EXPECT_EQ(50, output_height); - // Scale to multiple number with odd targe size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width, - &output_height)); - EXPECT_EQ(100, output_width); - EXPECT_EQ(50, output_height); - // Scale to multiple number with odd targe size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width, - &output_height)); - EXPECT_EQ(100, output_width); - EXPECT_EQ(50, output_height); - // Scale to odd size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width, - &output_height)); - EXPECT_EQ(151, output_width); - EXPECT_EQ(101, output_height); -} - -// Tests scaling without keeping the aspect ratio fixed. -TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) { - int output_width; - int output_height; - // Scale width only. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width, - &output_height)); - EXPECT_EQ(100, output_width); - EXPECT_EQ(100, output_height); - // Scale height only. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width, - &output_height)); - EXPECT_EQ(200, output_width); - EXPECT_EQ(200, output_height); - // Scale both dimensions. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width, - &output_height)); - EXPECT_EQ(150, output_width); - EXPECT_EQ(200, output_height); -} - -// Tests scale_to_multiple_of. -TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) { - int output_width; - int output_height; - // Set no targets, downscale to a multiple of 8. - MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width, - &output_height)); - EXPECT_EQ(96, output_width); - EXPECT_EQ(96, output_height); - // Set width target, downscale to a multiple of 8. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width, - &output_height)); - EXPECT_EQ(96, output_width); - EXPECT_EQ(96, output_height); - // Set height target, downscale to a multiple of 8. - MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width, - &output_height)); - EXPECT_EQ(200, output_width); - EXPECT_EQ(200, output_height); - // Set both targets, downscale to a multiple of 8. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width, - &output_height)); - EXPECT_EQ(144, output_width); - EXPECT_EQ(200, output_height); - // Doesn't throw error if keep aspect is true and downscale multiple is 2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width, - &output_height)); - EXPECT_EQ(400, output_width); - EXPECT_EQ(200, output_height); - // Throws error if keep aspect is true, but downscale multiple is not 2. - ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width, - &output_height), - testing::Not(testing::status::IsOk())); - // Downscaling to multiple ignored if multiple is less than 2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width, - &output_height)); - EXPECT_EQ(401, output_width); - EXPECT_EQ(201, output_height); -} - -} // namespace -} // namespace scale_image -} // namespace mediapipe diff --git a/mediapipe/calculators/image/set_alpha_calculator.cc b/mediapipe/calculators/image/set_alpha_calculator.cc deleted file mode 100644 index 87a661be6..000000000 --- a/mediapipe/calculators/image/set_alpha_calculator.cc +++ /dev/null @@ -1,473 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/image/set_alpha_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { - -namespace { - -constexpr char kInputFrameTag[] = "IMAGE"; -constexpr char kInputAlphaTag[] = "ALPHA"; -constexpr char kOutputFrameTag[] = "IMAGE"; - -constexpr char kInputFrameTagGpu[] = "IMAGE_GPU"; -constexpr char kInputAlphaTagGpu[] = "ALPHA_GPU"; -constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU"; - -constexpr int kNumChannelsRGBA = 4; - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -} // namespace - -// A calculator for setting the alpha channel of an RGBA image. -// -// The alpha channel can be set to a single value, or come from an image mask. -// If the input image has an alpha channel, it will be updated. -// If the input image doesn't have an alpha channel, one will be added. -// Adding alpha channel to a Grayscale (single channel) input is not suported. -// -// Inputs: -// One of the following two IMAGE tags: -// IMAGE: ImageFrame containing input image - RGB or RGBA. -// IMAGE_GPU: GpuBuffer containing input image - RGB or RGBA. -// -// ALPHA (optional): ImageFrame alpha mask to apply, -// can be any # of channels, only first channel used, -// must be same format as input -// ALPHA_GPU (optional): GpuBuffer alpha mask to apply, -// can be any # of channels, only first channel used, -// must be same format as input -// If ALPHA* input tag is not set, the 'alpha_value' option must be used. -// -// Output: -// One of the following two tags: -// IMAGE: An ImageFrame with alpha channel set - RGBA only. -// IMAGE_GPU: A GpuBuffer with alpha channel set - RGBA only. -// -// Options: -// alpha_value (optional): The alpha value to set to input image, [0-255], -// takes precedence over input mask. -// If alpha_value is not set, the ALPHA* input tag must be used. -// -// Notes: -// Either alpha_value option or ALPHA (or ALPHA_GPU) must be set. -// All CPU inputs must have the same image dimensions and data type. -// -class SetAlphaCalculator : public CalculatorBase { - public: - SetAlphaCalculator() = default; - ~SetAlphaCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - - absl::Status GlSetup(CalculatorContext* cc); - void GlRender(CalculatorContext* cc); - - mediapipe::SetAlphaCalculatorOptions options_; - float alpha_value_ = -1.f; - - bool use_gpu_ = false; - bool gpu_initialized_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint program_ = 0; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(SetAlphaCalculator); - -absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); - - bool use_gpu = false; - - if (cc->Inputs().HasTag(kInputFrameTag) && - cc->Inputs().HasTag(kInputFrameTagGpu)) { - return absl::InternalError("Cannot have multiple input images."); - } - if (cc->Inputs().HasTag(kInputFrameTagGpu) != - cc->Outputs().HasTag(kOutputFrameTagGpu)) { - return absl::InternalError("GPU output must have GPU input."); - } - - // Input image to add/edit alpha channel. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputFrameTagGpu)) { - cc->Inputs().Tag(kInputFrameTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputFrameTag)) { - cc->Inputs().Tag(kInputFrameTag).Set(); - } - - // Input alpha image mask (optional) -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputAlphaTagGpu)) { - cc->Inputs().Tag(kInputAlphaTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kInputAlphaTag)) { - cc->Inputs().Tag(kInputAlphaTag).Set(); - } - - // RGBA output image. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kOutputFrameTagGpu)) { - cc->Outputs().Tag(kOutputFrameTagGpu).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kOutputFrameTag)) { - cc->Outputs().Tag(kOutputFrameTag).Set(); - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status SetAlphaCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - - if (cc->Inputs().HasTag(kInputFrameTagGpu) && - cc->Outputs().HasTag(kOutputFrameTagGpu)) { -#if !MEDIAPIPE_DISABLE_GPU - use_gpu_ = true; -#else - RET_CHECK_FAIL() << "GPU processing not enabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - // Get global value from options (-1 if not set). - alpha_value_ = options_.alpha_value(); - if (use_gpu_) alpha_value_ /= 255.0; - - const bool use_image_mask = cc->Inputs().HasTag(kInputAlphaTag) || - cc->Inputs().HasTag(kInputAlphaTagGpu); - if (!((alpha_value_ >= 0) ^ use_image_mask)) - RET_CHECK_FAIL() << "Must use either image mask or options alpha value."; - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif - } // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status SetAlphaCalculator::Process(CalculatorContext* cc) { - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(GlSetup(cc)); - gpu_initialized_ = true; - } - MP_RETURN_IF_ERROR(RenderGpu(cc)); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(RenderCpu(cc)); - } - - return absl::OkStatus(); -} - -absl::Status SetAlphaCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - program_ = 0; - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kInputFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - - // Setup source image - const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get(); - const cv::Mat input_mat = mediapipe::formats::MatView(&input_frame); - if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) { - LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported"; - } - - // Setup destination image - auto output_frame = absl::make_unique( - ImageFormat::SRGBA, input_mat.cols, input_mat.rows); - cv::Mat output_mat = mediapipe::formats::MatView(output_frame.get()); - - const bool has_alpha_mask = cc->Inputs().HasTag(kInputAlphaTag) && - !cc->Inputs().Tag(kInputAlphaTag).IsEmpty(); - const bool use_alpa_mask = alpha_value_ < 0 && has_alpha_mask; - - // Setup alpha image and Update image in CPU. - if (use_alpa_mask) { - const auto& alpha_mask = cc->Inputs().Tag(kInputAlphaTag).Get(); - cv::Mat alpha_mat = mediapipe::formats::MatView(&alpha_mask); - RET_CHECK_EQ(input_mat.rows, alpha_mat.rows); - RET_CHECK_EQ(input_mat.cols, alpha_mat.cols); - - for (int i = 0; i < output_mat.rows; ++i) { - const uchar* in_ptr = input_mat.ptr(i); - uchar* alpha_ptr = alpha_mat.ptr(i); - uchar* out_ptr = output_mat.ptr(i); - for (int j = 0; j < output_mat.cols; ++j) { - const int out_idx = j * kNumChannelsRGBA; - const int in_idx = j * input_mat.channels(); - const int alpha_idx = j * alpha_mat.channels(); - out_ptr[out_idx + 0] = in_ptr[in_idx + 0]; - out_ptr[out_idx + 1] = in_ptr[in_idx + 1]; - out_ptr[out_idx + 2] = in_ptr[in_idx + 2]; - out_ptr[out_idx + 3] = alpha_ptr[alpha_idx + 0]; // channel 0 of mask - } - } - } else { - const uchar alpha_value = std::min(std::max(0.0f, alpha_value_), 255.0f); - for (int i = 0; i < output_mat.rows; ++i) { - const uchar* in_ptr = input_mat.ptr(i); - uchar* out_ptr = output_mat.ptr(i); - for (int j = 0; j < output_mat.cols; ++j) { - const int out_idx = j * kNumChannelsRGBA; - const int in_idx = j * input_mat.channels(); - out_ptr[out_idx + 0] = in_ptr[in_idx + 0]; - out_ptr[out_idx + 1] = in_ptr[in_idx + 1]; - out_ptr[out_idx + 2] = in_ptr[in_idx + 2]; - out_ptr[out_idx + 3] = alpha_value; // use value from options - } - } - } - - cc->Outputs() - .Tag(kOutputFrameTag) - .Add(output_frame.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) { - if (cc->Inputs().Tag(kInputFrameTagGpu).IsEmpty()) { - return absl::OkStatus(); - } -#if !MEDIAPIPE_DISABLE_GPU - // Setup source texture. - const auto& input_frame = - cc->Inputs().Tag(kInputFrameTagGpu).Get(); - if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 || - input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) { - LOG(ERROR) << "Only RGB or RGBA input image supported"; - } - auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); - - // Setup destination texture. - const int width = input_frame.width(), height = input_frame.height(); - auto output_texture = gpu_helper_.CreateDestinationTexture( - width, height, mediapipe::GpuBufferFormat::kBGRA32); - - const bool has_alpha_mask = cc->Inputs().HasTag(kInputAlphaTagGpu) && - !cc->Inputs().Tag(kInputAlphaTagGpu).IsEmpty(); - - // Setup alpha texture and Update image in GPU shader. - if (has_alpha_mask) { - const auto& alpha_mask = - cc->Inputs().Tag(kInputAlphaTagGpu).Get(); - auto alpha_texture = gpu_helper_.CreateSourceTexture(alpha_mask); - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_texture.name()); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, alpha_texture.name()); - GlRender(cc); // use channel 0 of mask - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - alpha_texture.Release(); - } else { - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_texture.name()); - GlRender(cc); // use value from options - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - } - glFlush(); - - // Send out image as GPU packet. - auto output_frame = output_texture.GetFrame(); - cc->Outputs() - .Tag(kOutputFrameTagGpu) - .Add(output_frame.release(), cc->InputTimestamp()); - - // Cleanup - input_texture.Release(); - output_texture.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void SetAlphaCalculator::GlRender(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(program_); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); - -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status SetAlphaCalculator::GlSetup(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Shader to overlay a texture onto another when overlay is non-zero. - // TODO split into two shaders to handle alpha_value<0 separately - const GLchar* frag_src = GLES_VERSION_COMPAT - R"( - #if __VERSION__ < 130 - #define in varying - #endif // __VERSION__ < 130 - - #ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; - #else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; - #endif // defined(GL_ES) - - in vec2 sample_coordinate; - uniform sampler2D input_frame; - uniform sampler2D alpha_mask; - uniform float alpha_value; - - void main() { - vec3 image_pix = texture2D(input_frame, sample_coordinate).rgb; - float alpha = alpha_value; - if (alpha_value < 0.0) alpha = texture2D(alpha_mask, sample_coordinate).r; - vec4 out_pix = vec4(image_pix, alpha); - fragColor = out_pix; - } - )"; - - // Create shader program and set parameters. - mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src, - NUM_ATTRIBUTES, (const GLchar**)&attr_name[0], - attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); - glUniform1i(glGetUniformLocation(program_, "alpha_mask"), 2); - glUniform1f(glGetUniformLocation(program_, "alpha_value"), alpha_value_); - -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/set_alpha_calculator.proto b/mediapipe/calculators/image/set_alpha_calculator.proto deleted file mode 100644 index 3f07ddb31..000000000 --- a/mediapipe/calculators/image/set_alpha_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message SetAlphaCalculatorOptions { - extend CalculatorOptions { - optional SetAlphaCalculatorOptions ext = 250949799; - } - - // The value to set the alpha channel to (0-255). - // This option is ignored when set to -1 (use image mask instead). - optional sint32 alpha_value = 1 [default = -1]; -} diff --git a/mediapipe/calculators/image/sobel_edges_calculator.cc b/mediapipe/calculators/image/sobel_edges_calculator.cc deleted file mode 100644 index 6154a246b..000000000 --- a/mediapipe/calculators/image/sobel_edges_calculator.cc +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/gl_simple_calculator.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -namespace mediapipe { - -// Applies the Sobel filter to an image. Expects a grayscale image stored as -// RGB, like LuminanceCalculator outputs. -// See GlSimpleCalculatorBase for inputs, outputs and input side packets. -class SobelEdgesCalculator : public GlSimpleCalculator { - public: - absl::Status GlSetup() override; - absl::Status GlRender(const GlTexture& src, const GlTexture& dst) override; - absl::Status GlTeardown() override; - - private: - GLuint program_ = 0; - GLint frame_; - GLint pixel_w_; - GLint pixel_h_; -}; -REGISTER_CALCULATOR(SobelEdgesCalculator); - -absl::Status SobelEdgesCalculator::GlSetup() { - // Load vertex and fragment shaders - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "vertexPosition", - "vertexTextureCoordinate", - }; - - const GLchar* vert_src = GLES_VERSION_COMPAT - R"( -#if __VERSION__ < 130 - #define in attribute - #define out varying -#endif // __VERSION__ < 130 - - in vec4 vertexPosition; - in vec4 vertexTextureCoordinate; - - // width of a pixel in normalized texture coordinates (0..1) - uniform highp float pixelW; - - // height of a pixel in normalized texture coordinates (0..1) - uniform highp float pixelH; - - // Dependent texture reads (i.e. texture reads where texture coordinates - // are computed in the fragment shader) are slow on pre-ES 3.0 hardware. - // Avoid them by computing all texture coordinates in the vertex shader. - - // iOS OGLES performance guide: https://developer.apple.com/library/ios/documentation/3DDrawing/Conceptual/OpenGLES_ProgrammingGuide/BestPracticesforShaders/BestPracticesforShaders.html - - // Code for coordinates: u = up, d = down, l = left, r = right, c = center. - // Horizontal coordinate first, then vertical. - out vec2 luTexCoord; - out vec2 lcTexCoord; - out vec2 ldTexCoord; - - out vec2 cuTexCoord; -// out vec2 ccTexCoord; - out vec2 cdTexCoord; - - out vec2 ruTexCoord; - out vec2 rcTexCoord; - out vec2 rdTexCoord; - - void main() { - gl_Position = vertexPosition; - - vec2 right = vec2(pixelW, 0.0); - vec2 up = vec2(0.0, pixelH); - - lcTexCoord = vertexTextureCoordinate.xy - right; - luTexCoord = lcTexCoord + up; - ldTexCoord = lcTexCoord - up; - - vec2 ccTexCoord = vertexTextureCoordinate.xy; - cuTexCoord = ccTexCoord + up; - cdTexCoord = ccTexCoord - up; - - rcTexCoord = vertexTextureCoordinate.xy + right; - ruTexCoord = rcTexCoord + up; - rdTexCoord = rcTexCoord - up; - } - )"; - const GLchar* frag_src = GLES_VERSION_COMPAT - R"( -#if __VERSION__ < 130 - #define in varying -#endif // __VERSION__ < 130 - -#ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; -#else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; -#endif // defined(GL_ES) - - in vec2 luTexCoord; - in vec2 lcTexCoord; - in vec2 ldTexCoord; - - in vec2 cuTexCoord; -// in vec2 ccTexCoord; - in vec2 cdTexCoord; - - in vec2 ruTexCoord; - in vec2 rcTexCoord; - in vec2 rdTexCoord; - - uniform sampler2D inputImage; - - void main() { - float luPx = texture2D(inputImage, luTexCoord).r; - float lcPx = texture2D(inputImage, lcTexCoord).r; - float ldPx = texture2D(inputImage, ldTexCoord).r; - - float cuPx = texture2D(inputImage, cuTexCoord).r; -// float ccPx = texture2D(inputImage, ccTexCoord).r; - float cdPx = texture2D(inputImage, cdTexCoord).r; - - float ruPx = texture2D(inputImage, ruTexCoord).r; - float rcPx = texture2D(inputImage, rcTexCoord).r; - float rdPx = texture2D(inputImage, rdTexCoord).r; - - float h = -luPx - 2.0 * lcPx - ldPx + ruPx + 2.0 * rcPx + rdPx; - float v = -luPx - 2.0 * cuPx - ruPx + ldPx + 2.0 * cdPx + rdPx; - - float mag = length(vec2(h, v)); - - fragColor = vec4(vec3(mag), 1.0); - } - )"; - - // shader program - GlhCreateProgram(vert_src, frag_src, NUM_ATTRIBUTES, - (const GLchar**)&attr_name[0], attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - frame_ = glGetUniformLocation(program_, "inputImage"); - pixel_w_ = glGetUniformLocation(program_, "pixelW"); - pixel_h_ = glGetUniformLocation(program_, "pixelH"); - return absl::OkStatus(); -} - -absl::Status SobelEdgesCalculator::GlRender(const GlTexture& src, - const GlTexture& dst) { - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const float texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(program_); - glUniform1i(frame_, 1); - - // parameters - glUniform1i(frame_, 1); - glUniform1f(pixel_w_, 1.0 / src.width()); - glUniform1f(pixel_h_, 1.0 / src.height()); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); - - return absl::OkStatus(); -} - -absl::Status SobelEdgesCalculator::GlTeardown() { - if (program_) { - glDeleteProgram(program_); - program_ = 0; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/image/testdata/BUILD b/mediapipe/calculators/image/testdata/BUILD deleted file mode 100644 index da192b513..000000000 --- a/mediapipe/calculators/image/testdata/BUILD +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -licenses(["notice"]) - -filegroup( - name = "test_images", - srcs = [ - "dino.jpg", - "dino_quality_50.jpg", - "dino_quality_80.jpg", - "front_camera_pixel2.jpg", - ], - visibility = ["//visibility:public"], -) diff --git a/mediapipe/calculators/image/testdata/dino.jpg b/mediapipe/calculators/image/testdata/dino.jpg deleted file mode 100644 index df2dfc693..000000000 Binary files a/mediapipe/calculators/image/testdata/dino.jpg and /dev/null differ diff --git a/mediapipe/calculators/image/testdata/dino_quality_50.jpg b/mediapipe/calculators/image/testdata/dino_quality_50.jpg deleted file mode 100644 index de8af0dd4..000000000 Binary files a/mediapipe/calculators/image/testdata/dino_quality_50.jpg and /dev/null differ diff --git a/mediapipe/calculators/image/testdata/dino_quality_80.jpg b/mediapipe/calculators/image/testdata/dino_quality_80.jpg deleted file mode 100644 index eb7aa7b25..000000000 Binary files a/mediapipe/calculators/image/testdata/dino_quality_80.jpg and /dev/null differ diff --git a/mediapipe/calculators/image/testdata/front_camera_pixel2.jpg b/mediapipe/calculators/image/testdata/front_camera_pixel2.jpg deleted file mode 100644 index c2164c415..000000000 Binary files a/mediapipe/calculators/image/testdata/front_camera_pixel2.jpg and /dev/null differ diff --git a/mediapipe/calculators/internal/BUILD b/mediapipe/calculators/internal/BUILD deleted file mode 100644 index 54b6c20f1..000000000 --- a/mediapipe/calculators/internal/BUILD +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -package(default_visibility = ["//visibility:private"]) - -proto_library( - name = "callback_packet_calculator_proto", - srcs = ["callback_packet_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "callback_packet_calculator_cc_proto", - srcs = ["callback_packet_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":callback_packet_calculator_proto"], -) - -cc_library( - name = "callback_packet_calculator", - srcs = ["callback_packet_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":callback_packet_calculator_cc_proto", - "//mediapipe/framework:calculator_base", - "//mediapipe/framework:calculator_registry", - "//mediapipe/framework:output_side_packet", - ], - alwayslink = 1, -) diff --git a/mediapipe/calculators/internal/callback_packet_calculator.cc b/mediapipe/calculators/internal/callback_packet_calculator.cc deleted file mode 100644 index cc153483e..000000000 --- a/mediapipe/calculators/internal/callback_packet_calculator.cc +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT -#include "mediapipe/framework/calculator_base.h" -#include "mediapipe/framework/calculator_registry.h" -#include "mediapipe/framework/output_side_packet.h" - -namespace mediapipe { - -namespace { - -// Callback function for writing a packet to a vector. The output is before the -// input since std::bind fills arguments from left to right (and only -// dumped_data is filled by std::bind). -void DumpToVector(std::vector* dumped_data, const Packet& packet) { - dumped_data->push_back(packet); -} - -// Callback function for saving the Timestamp::PostStream() packet. -// The output is before the input since std::bind fills arguments from left to -// right (and only post_stream_packet is filled by std::bind). -void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) { - if (packet.Timestamp() == Timestamp::PostStream()) { - *post_stream_packet = packet; - } -} -} // namespace - -// Creates a callback which takes a packet and stores it either in a -// vector of packets or stores only the packet at PostStream timestamp. -// The kind of callback is controlled by an option. The callback is -// a std::function and is directly usable by CallbackCalculator. -// Since the options for the packet generator include a serialized pointer -// value, the resulting callback is only valid on the original machine -// while that pointer is still alive. -class CallbackPacketCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - switch (options.type()) { - case CallbackPacketCalculatorOptions::VECTOR_PACKET: - case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: - cc->OutputSidePackets() - .Index(0) - .Set>(); - break; - default: - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Invalid type of callback to produce."; - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const auto& options = cc->Options(); - void* ptr; - if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Stored pointer value in options is invalid."; - } - switch (options.type()) { - case CallbackPacketCalculatorOptions::VECTOR_PACKET: - cc->OutputSidePackets().Index(0).Set( - MakePacket>(std::bind( - &DumpToVector, reinterpret_cast*>(ptr), - std::placeholders::_1))); - break; - case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: - cc->OutputSidePackets().Index(0).Set( - MakePacket>( - std::bind(&DumpPostStreamPacket, reinterpret_cast(ptr), - std::placeholders::_1))); - break; - default: - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Invalid type to dump into."; - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(CallbackPacketCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/internal/callback_packet_calculator.proto b/mediapipe/calculators/internal/callback_packet_calculator.proto deleted file mode 100644 index 6a5cfb05a..000000000 --- a/mediapipe/calculators/internal/callback_packet_calculator.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message CallbackPacketCalculatorOptions { - extend CalculatorOptions { - optional CallbackPacketCalculatorOptions ext = 245965803; - } - - enum PointerType { - UNKNOWN = 0; - VECTOR_PACKET = 1; - POST_STREAM_PACKET = 2; - } - - // The type of the data pointer that the callback will put data into. - optional PointerType type = 1; - // The location of the data stored as a string printed with - // snprintf(address, sizeof(address), "%p", pointer). - // This calculator only produces a reasonable callback if it is - // constructed on the same machine as the original pointer was created on and - // that pointer is still alive. - optional bytes pointer = 2; -} diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD deleted file mode 100644 index 3979def5e..000000000 --- a/mediapipe/calculators/tensor/BUILD +++ /dev/null @@ -1,828 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("@bazel_skylib//lib:selects.bzl", "selects") -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") -load( - "//mediapipe/framework/tool:mediapipe_graph.bzl", - "mediapipe_binary_graph", -) -load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") -load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -selects.config_setting_group( - name = "compute_shader_unavailable", - match_any = [ - "//mediapipe/gpu:disable_gpu", - ], -) - -mediapipe_proto_library( - name = "inference_calculator_proto", - srcs = ["inference_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "inference_calculator_interface", - srcs = ["inference_calculator.cc"], - hdrs = ["inference_calculator.h"], - copts = select({ - # TODO: fix tensor.h not to require this, if possible - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - ":inference_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/tool:subgraph_expansion", - "//mediapipe/util/tflite:config", - "//mediapipe/util/tflite:tflite_model_loader", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - ], - alwayslink = 1, -) - -cc_library( - name = "inference_calculator_gl", - srcs = ["inference_calculator_gl.cc"], - tags = ["nomac"], # config problem with cpuinfo via TF - deps = [ - "inference_calculator_interface", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/util/tflite:tflite_gpu_runner", - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - ], - alwayslink = 1, -) - -cc_library( - name = "inference_calculator_metal", - srcs = ["inference_calculator_metal.cc"], - copts = [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - linkopts = [ - "-framework CoreVideo", - "-framework MetalKit", - ], - tags = ["ios"], - deps = [ - "inference_calculator_interface", - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/objc:mediapipe_framework_ios", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert", - ], - alwayslink = 1, -) - -cc_library( - name = "inference_calculator_cpu", - srcs = [ - "inference_calculator_cpu.cc", - ], - copts = select({ - # TODO: fix tensor.h not to require this, if possible - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - ":inference_calculator_interface", - "@com_google_absl//absl/memory", - "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", - ] + select({ - "//conditions:default": [ - "//mediapipe/util:cpu_util", - ], - }) + select({ - "//conditions:default": [], - "//mediapipe:android": ["@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate"], - }), - alwayslink = 1, -) - -cc_library( - name = "inference_calculator_gl_if_compute_shader_available", - deps = select({ - ":compute_shader_unavailable": [], - "//conditions:default": [":inference_calculator_gl"], - }), -) - -cc_library( - name = "inference_calculator", - visibility = ["//visibility:public"], - deps = [ - ":inference_calculator_interface", - ":inference_calculator_cpu", - ] + select({ - "//conditions:default": [":inference_calculator_gl_if_compute_shader_available"], - "//mediapipe:ios": [":inference_calculator_metal"], - }), - alwayslink = 1, -) - -mediapipe_proto_library( - name = "tensor_converter_calculator_proto", - srcs = ["tensor_converter_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "tensor_converter_calculator", - srcs = ["tensor_converter_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - features = ["-layering_check"], # allow depending on tensor_converter_calculator_gpu_deps - linkopts = select({ - "//mediapipe:apple": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensor_converter_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework:port", - "//mediapipe/util:resource_util", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": ["tensor_converter_calculator_gpu_deps"], - }), - alwayslink = 1, -) - -cc_library( - name = "tensor_converter_calculator_gpu_deps", - deps = select({ - "//mediapipe:android": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - ], - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/objc:mediapipe_framework_ios", - ], - "//mediapipe:macos": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:shader_util", - "//mediapipe/gpu:gpu_buffer", - ], - }), -) - -cc_test( - name = "tensor_converter_calculator_test", - srcs = ["tensor_converter_calculator_test.cc"], - deps = [ - ":tensor_converter_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -mediapipe_proto_library( - name = "tensors_to_detections_calculator_proto", - srcs = ["tensors_to_detections_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "tensors_to_detections_calculator", - srcs = ["tensors_to_detections_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - features = ["-layering_check"], # allow depending on tensors_to_detections_calculator_gpu_deps - linkopts = select({ - "//mediapipe:apple": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensors_to_detections_calculator_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/api2:node", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:port", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/formats/object_detection:anchor_cc_proto", - "//mediapipe/framework/port:ret_check", - ] + select({ - ":compute_shader_unavailable": [], - "//conditions:default": [":tensors_to_detections_calculator_gpu_deps"], - }), - alwayslink = 1, -) - -cc_library( - name = "tensors_to_detections_calculator_gpu_deps", - deps = select({ - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:MPPMetalHelper", - ], - "//mediapipe:macos": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - ], - }), -) - -mediapipe_proto_library( - name = "tensors_to_landmarks_calculator_proto", - srcs = ["tensors_to_landmarks_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "tensors_to_landmarks_calculator", - srcs = ["tensors_to_landmarks_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensors_to_landmarks_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "tensors_to_floats_calculator_proto", - srcs = ["tensors_to_floats_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "tensors_to_floats_calculator", - srcs = ["tensors_to_floats_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensors_to_floats_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -cc_test( - name = "tensors_to_floats_calculator_test", - srcs = ["tensors_to_floats_calculator_test.cc"], - deps = [ - ":tensors_to_floats_calculator", - ":tensors_to_floats_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/memory", - "@com_google_googletest//:gtest_main", - ], -) - -cc_library( - name = "tensors_to_classification_calculator", - srcs = ["tensors_to_classification_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensors_to_classification_calculator_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/formats:tensor", - "//mediapipe/util:resource_util", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -mediapipe_proto_library( - name = "tensors_to_classification_calculator_proto", - srcs = ["tensors_to_classification_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_test( - name = "tensors_to_classification_calculator_test", - srcs = ["tensors_to_classification_calculator_test.cc"], - data = ["testdata/labelmap.txt"], - deps = [ - ":tensors_to_classification_calculator", - ":tensors_to_classification_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/memory", - "@com_google_googletest//:gtest_main", - ], -) - -cc_library( - name = "image_to_tensor_calculator", - srcs = ["image_to_tensor_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - features = ["-layering_check"], # allow depending on image_to_tensor_calculator_gpu_deps - visibility = ["//visibility:public"], - deps = [ - ":image_to_tensor_calculator_cc_proto", - ":image_to_tensor_converter", - ":image_to_tensor_converter_opencv", - ":image_to_tensor_utils", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:port", - "//mediapipe/gpu:gpu_origin_cc_proto", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [":image_to_tensor_calculator_gpu_deps"], - }), - alwayslink = 1, -) - -cc_library( - name = "image_to_tensor_calculator_gpu_deps", - deps = select({ - "//mediapipe:android": [ - ":image_to_tensor_converter_gl_buffer", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - ], - "//mediapipe:apple": [ - ":image_to_tensor_converter_metal", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/gpu:gpu_buffer", - ], - "//conditions:default": [ - ":image_to_tensor_converter_gl_buffer", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - ], - }), -) - -mediapipe_proto_library( - name = "image_to_tensor_calculator_proto", - srcs = ["image_to_tensor_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/gpu:gpu_origin_proto", - ], -) - -cc_test( - name = "image_to_tensor_calculator_test", - srcs = ["image_to_tensor_calculator_test.cc"], - data = [ - "testdata/image_to_tensor/input.jpg", - "testdata/image_to_tensor/large_sub_rect.png", - "testdata/image_to_tensor/large_sub_rect_border_zero.png", - "testdata/image_to_tensor/large_sub_rect_keep_aspect.png", - "testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png", - "testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png", - "testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png", - "testdata/image_to_tensor/medium_sub_rect_keep_aspect.png", - "testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png", - "testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png", - "testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png", - "testdata/image_to_tensor/medium_sub_rect_with_rotation.png", - "testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png", - "testdata/image_to_tensor/noop_except_range.png", - ], - deps = [ - ":image_to_tensor_calculator", - ":image_to_tensor_converter", - ":image_to_tensor_utils", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "image_to_tensor_converter", - hdrs = ["image_to_tensor_converter.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":image_to_tensor_utils", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:statusor", - ], -) - -cc_library( - name = "image_to_tensor_converter_opencv", - srcs = ["image_to_tensor_converter_opencv.cc"], - hdrs = ["image_to_tensor_converter_opencv.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - ":image_to_tensor_converter", - ":image_to_tensor_utils", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - ], -) - -cc_library( - name = "image_to_tensor_converter_gl_buffer", - srcs = ["image_to_tensor_converter_gl_buffer.cc"], - hdrs = ["image_to_tensor_converter_gl_buffer.h"], - deps = ["//mediapipe/framework:port"] + select({ - "//mediapipe:apple": [], - "//conditions:default": [ - ":image_to_tensor_converter", - ":image_to_tensor_converter_gl_utils", - ":image_to_tensor_utils", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/framework/formats:image", - "//mediapipe/gpu:gpu_buffer_format", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:types", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:command_queue", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_call", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:request_gpu_info", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:variable", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util", - ], - }), -) - -cc_library( - name = "image_to_tensor_converter_gl_texture", - srcs = ["image_to_tensor_converter_gl_texture.cc"], - hdrs = ["image_to_tensor_converter_gl_texture.h"], - deps = ["//mediapipe/framework:port"] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - ":image_to_tensor_converter", - ":image_to_tensor_converter_gl_utils", - ":image_to_tensor_utils", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/framework/formats:image", - "//mediapipe/gpu:shader_util", - ], - }), -) - -cc_library( - name = "image_to_tensor_converter_gl_utils", - srcs = ["image_to_tensor_converter_gl_utils.cc"], - hdrs = ["image_to_tensor_converter_gl_utils.h"], - deps = ["//mediapipe/framework:port"] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_base", - "//mediapipe/gpu:gl_context", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - ], - }), -) - -cc_library( - name = "image_to_tensor_converter_metal", - srcs = ["image_to_tensor_converter_metal.cc"], - hdrs = ["image_to_tensor_converter_metal.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:apple": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - deps = ["//mediapipe/framework:port"] + select({ - "//mediapipe:apple": [ - ":image_to_tensor_converter", - ":image_to_tensor_utils", - "//mediapipe/gpu:MPPMetalHelper", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework/formats:image", - "//mediapipe/gpu:gpu_buffer_format", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:types", - ], - "//conditions:default": [], - }), -) - -cc_library( - name = "image_to_tensor_utils", - srcs = ["image_to_tensor_utils.cc"], - hdrs = ["image_to_tensor_utils.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:statusor", - "@com_google_absl//absl/types:optional", - ], -) - -cc_test( - name = "image_to_tensor_utils_test", - srcs = ["image_to_tensor_utils_test.cc"], - deps = [ - ":image_to_tensor_utils", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - ], -) - -# Copied from /mediapipe/calculators/tflite/BUILD -selects.config_setting_group( - name = "gpu_inference_disabled", - match_any = [ - "//mediapipe/gpu:disable_gpu", - ], -) - -mediapipe_proto_library( - name = "tensors_to_segmentation_calculator_proto", - srcs = ["tensors_to_segmentation_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/gpu:gpu_origin_proto", - ], -) - -cc_library( - name = "tensors_to_segmentation_calculator", - srcs = ["tensors_to_segmentation_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensors_to_segmentation_calculator_cc_proto", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:port", - "//mediapipe/util:resource_util", - "@org_tensorflow//tensorflow/lite:framework", - "//mediapipe/gpu:gpu_origin_cc_proto", - "//mediapipe/framework/port:statusor", - ] + selects.with_or({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:shader_util", - ], - }) + selects.with_or({ - ":gpu_inference_disabled": [], - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:MPPMetalHelper", - ], - "//conditions:default": [ - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl/converters:util", - ], - }), - alwayslink = 1, -) diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc deleted file mode 100644 index f681ab661..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gpu_origin.pb.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gpu_buffer.h" - -#if MEDIAPIPE_METAL_ENABLED -#include "mediapipe/calculators/tensor/image_to_tensor_converter_metal.h" -#include "mediapipe/gpu/MPPMetalHelper.h" -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#else -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // MEDIAPIPE_METAL_ENABLED - -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { -namespace api2 { - -#if MEDIAPIPE_DISABLE_GPU -// Just a placeholder to not have to depend on mediapipe::GpuBuffer. -using GpuBuffer = AnyType; -#else -using GpuBuffer = mediapipe::GpuBuffer; -#endif // MEDIAPIPE_DISABLE_GPU - -// Converts image into Tensor, possibly with cropping, resizing and -// normalization, according to specified inputs and options. -// -// Inputs: -// IMAGE - Image[ImageFormat::SRGB / SRGBA, GpuBufferFormat::kBGRA32] or -// ImageFrame [ImageFormat::SRGB/SRGBA] (for backward compatibility -// with existing graphs that use IMAGE for ImageFrame input) -// IMAGE_GPU - GpuBuffer [GpuBufferFormat::kBGRA32] -// Image to extract from. -// -// Note: -// - One and only one of IMAGE and IMAGE_GPU should be specified. -// - IMAGE input of type Image is processed on GPU if the data is already on -// GPU (i.e., Image::UsesGpu() returns true), or otherwise processed on CPU. -// - IMAGE input of type ImageFrame is always processed on CPU. -// - IMAGE_GPU input (of type GpuBuffer) is always processed on GPU. -// -// NORM_RECT - NormalizedRect @Optional -// Describes region of image to extract. -// @Optional: rect covering the whole image is used if not specified. -// -// Outputs: -// TENSORS - std::vector -// Vector containing a single Tensor populated with an extrated RGB image. -// MATRIX - std::array @Optional -// An std::array representing a 4x4 row-major-order matrix which -// can be used to map a point on the output tensor to a point on the input -// image. -// LETTERBOX_PADDING - std::array @Optional -// An std::array representing the letterbox padding from the 4 -// sides ([left, top, right, bottom]) of the output image, normalized to -// [0.f, 1.f] by the output dimensions. The padding values are non-zero only -// when the "keep_aspect_ratio" is true. -// -// For instance, when the input image is 10x10 (width x height) and the -// output dimensions specified in the calculator option are 20x40 and -// "keep_aspect_ratio" is true, the calculator scales the input image to -// 20x20 and places it in the middle of the output image with an equal -// padding of 10 pixels at the top and the bottom. The resulting array is -// therefore [0.f, 0.25f, 0.f, 0.25f] (10/40 = 0.25f). -// -// Example: -// node { -// calculator: "ImageToTensorCalculator" -// input_stream: "IMAGE:image" # or "IMAGE_GPU:image" -// input_stream: "NORM_RECT:roi" -// output_stream: "TENSORS:tensors" -// output_stream: "MATRIX:matrix" -// options { -// [mediapipe.ImageToTensorCalculatorOptions.ext] { -// output_tensor_width: 256 -// output_tensor_height: 256 -// keep_aspect_ratio: false -// output_tensor_float_range { -// min: 0.0 -// max: 1.0 -// } -// # gpu_origin: CONVENTIONAL # or TOP_LEFT -// } -// } -// } -class ImageToTensorCalculator : public Node { - public: - static constexpr Input< - OneOf>::Optional kIn{"IMAGE"}; - static constexpr Input::Optional kInGpu{"IMAGE_GPU"}; - static constexpr Input::Optional kInNormRect{ - "NORM_RECT"}; - static constexpr Output> kOutTensors{"TENSORS"}; - static constexpr Output>::Optional kOutLetterboxPadding{ - "LETTERBOX_PADDING"}; - static constexpr Output>::Optional kOutMatrix{"MATRIX"}; - - MEDIAPIPE_NODE_CONTRACT(kIn, kInGpu, kInNormRect, kOutTensors, - kOutLetterboxPadding, kOutMatrix); - - static absl::Status UpdateContract(CalculatorContract* cc) { - const auto& options = - cc->Options(); - - RET_CHECK(options.has_output_tensor_float_range()) - << "Output tensor range is required."; - RET_CHECK_LT(options.output_tensor_float_range().min(), - options.output_tensor_float_range().max()) - << "Valid output tensor range is required."; - RET_CHECK_GT(options.output_tensor_width(), 0) - << "Valid output tensor width is required."; - RET_CHECK_GT(options.output_tensor_height(), 0) - << "Valid output tensor height is required."; - - RET_CHECK(kIn(cc).IsConnected() ^ kInGpu(cc).IsConnected()) - << "One and only one of IMAGE and IMAGE_GPU input is expected."; - -#if MEDIAPIPE_DISABLE_GPU - if (kInGpu(cc).IsConnected()) { - return absl::UnimplementedError( - "GPU processing is disabled in build flags"); - } -#else // !MEDIAPIPE_DISABLE_GPU -#if MEDIAPIPE_METAL_ENABLED - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#else - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // MEDIAPIPE_METAL_ENABLED -#endif // MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) { - options_ = cc->Options(); - output_width_ = options_.output_tensor_width(); - output_height_ = options_.output_tensor_height(); - range_min_ = options_.output_tensor_float_range().min(); - range_max_ = options_.output_tensor_float_range().max(); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) { - if ((kIn(cc).IsConnected() && kIn(cc).IsEmpty()) || - (kInGpu(cc).IsConnected() && kInGpu(cc).IsEmpty())) { - // Timestamp bound update happens automatically. - return absl::OkStatus(); - } - - absl::optional norm_rect; - if (kInNormRect(cc).IsConnected()) { - if (kInNormRect(cc).IsEmpty()) { - // Timestamp bound update happens automatically. (See Open().) - return absl::OkStatus(); - } - norm_rect = *kInNormRect(cc); - if (norm_rect->width() == 0 && norm_rect->height() == 0) { - // WORKAROUND: some existing graphs may use sentinel rects {width=0, - // height=0, ...} quite often and calculator has to handle them - // gracefully by updating timestamp bound instead of returning failure. - // Timestamp bound update happens automatically. (See Open().) - // NOTE: usage of sentinel rects should be avoided. - DLOG(WARNING) - << "Updating timestamp bound in response to a sentinel rect"; - return absl::OkStatus(); - } - } - - ASSIGN_OR_RETURN(auto image, GetInputImage(cc)); - const Size size{image->width(), image->height()}; - RotatedRect roi = GetRoi(size.width, size.height, norm_rect); - ASSIGN_OR_RETURN(auto padding, PadRoi(options_.output_tensor_width(), - options_.output_tensor_height(), - options_.keep_aspect_ratio(), &roi)); - if (kOutLetterboxPadding(cc).IsConnected()) { - kOutLetterboxPadding(cc).Send(padding); - } - if (kOutMatrix(cc).IsConnected()) { - std::array matrix; - GetRotatedSubRectToRectTransformMatrix(roi, size.width, size.height, - /*flip_horizontaly=*/false, - &matrix); - kOutMatrix(cc).Send(std::move(matrix)); - } - - // Lazy initialization of the GPU or CPU converter. - MP_RETURN_IF_ERROR(InitConverterIfNecessary(cc, image->UsesGpu())); - - ASSIGN_OR_RETURN(Tensor tensor, - (image->UsesGpu() ? gpu_converter_ : cpu_converter_) - ->Convert(*image, roi, {output_width_, output_height_}, - range_min_, range_max_)); - - auto result = std::make_unique>(); - result->push_back(std::move(tensor)); - kOutTensors(cc).Send(std::move(result)); - - return absl::OkStatus(); - } - - private: - bool DoesGpuInputStartAtBottom() { - return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; - } - - BorderMode GetBorderMode() { - switch (options_.border_mode()) { - case mediapipe:: - ImageToTensorCalculatorOptions_BorderMode_BORDER_UNSPECIFIED: - return BorderMode::kReplicate; - case mediapipe::ImageToTensorCalculatorOptions_BorderMode_BORDER_ZERO: - return BorderMode::kZero; - case mediapipe:: - ImageToTensorCalculatorOptions_BorderMode_BORDER_REPLICATE: - return BorderMode::kReplicate; - } - } - - absl::StatusOr> GetInputImage( - CalculatorContext* cc) { - if (kIn(cc).IsConnected()) { - const auto& packet = kIn(cc).packet(); - return kIn(cc).Visit( - [&packet](const mediapipe::Image&) { - return SharedPtrWithPacket(packet); - }, - [&packet](const mediapipe::ImageFrame&) { - return std::make_shared( - std::const_pointer_cast( - SharedPtrWithPacket(packet))); - }); - } else { // if (kInGpu(cc).IsConnected()) -#if !MEDIAPIPE_DISABLE_GPU - const GpuBuffer& input = *kInGpu(cc); - // A shallow copy is okay since the resulting 'image' object is local in - // Process(), and thus never outlives 'input'. - return std::make_shared(input); -#else - return absl::UnimplementedError( - "GPU processing is disabled in build flags"); -#endif // !MEDIAPIPE_DISABLE_GPU - } - } - - absl::Status InitConverterIfNecessary(CalculatorContext* cc, bool use_gpu) { - // Lazy initialization of the GPU or CPU converter. - if (use_gpu) { - if (!gpu_converter_) { -#if !MEDIAPIPE_DISABLE_GPU -#if MEDIAPIPE_METAL_ENABLED - ASSIGN_OR_RETURN(gpu_converter_, - CreateMetalConverter(cc, GetBorderMode())); -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - ASSIGN_OR_RETURN(gpu_converter_, - CreateImageToGlBufferTensorConverter( - cc, DoesGpuInputStartAtBottom(), GetBorderMode())); -#else - ASSIGN_OR_RETURN(gpu_converter_, - CreateImageToGlTextureTensorConverter( - cc, DoesGpuInputStartAtBottom(), GetBorderMode())); -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU - } - } else { - if (!cpu_converter_) { - ASSIGN_OR_RETURN(cpu_converter_, - CreateOpenCvConverter(cc, GetBorderMode())); - } - } - return absl::OkStatus(); - } - - std::unique_ptr gpu_converter_; - std::unique_ptr cpu_converter_; - mediapipe::ImageToTensorCalculatorOptions options_; - int output_width_ = 0; - int output_height_ = 0; - float range_min_ = 0.0f; - float range_max_ = 1.0f; -}; - -MEDIAPIPE_REGISTER_NODE(ImageToTensorCalculator); - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.proto b/mediapipe/calculators/tensor/image_to_tensor_calculator.proto deleted file mode 100644 index 0451dc51f..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.proto +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/gpu/gpu_origin.proto"; - -message ImageToTensorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional ImageToTensorCalculatorOptions ext = 334361939; - } - - // Range of float values [min, max]. - // min, must be strictly less than max. - message FloatRange { - optional float min = 1; - optional float max = 2; - } - - // Pixel extrapolation methods. See @border_mode. - enum BorderMode { - BORDER_UNSPECIFIED = 0; - BORDER_ZERO = 1; - BORDER_REPLICATE = 2; - } - - optional int32 output_tensor_width = 1; - optional int32 output_tensor_height = 2; - - // If true, image region will be extracted and copied into tensor keeping - // region aspect ratio, which usually results in letterbox padding. Otherwise, - // if false, image region is stretched to fill output tensor fully. - optional bool keep_aspect_ratio = 3; - - // Output tensor element range/type image pixels are converted to. - oneof range { - FloatRange output_tensor_float_range = 4; - } - - // For CONVENTIONAL mode for OpenGL, input image starts at bottom and needs - // to be flipped vertically as tensors are expected to start at top. - // (DEFAULT or unset interpreted as CONVENTIONAL.) - optional GpuOrigin.Mode gpu_origin = 5; - - // Pixel extrapolation method. - // When converting image to tensor it may happen that tensor needs to read - // pixels outside image boundaries. Border mode helps to specify how such - // pixels will be calculated. - // - // BORDER_REPLICATE is used by default. - optional BorderMode border_mode = 6; -} diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc deleted file mode 100644 index 275c33559..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/flags/flag.h" -#include "absl/memory/memory.h" -#include "absl/strings/substitute.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -cv::Mat GetRgb(absl::string_view path) { - cv::Mat bgr = cv::imread(file::JoinPath("./", path)); - cv::Mat rgb; - cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGB); - return rgb; -} - -cv::Mat GetRgba(absl::string_view path) { - cv::Mat bgr = cv::imread(file::JoinPath("./", path)); - cv::Mat rgb; - cv::cvtColor(bgr, rgb, cv::COLOR_BGR2RGBA); - return rgb; -} - -// Image to tensor test template. -// No processing/assertions should be done after the function is invoked. -void RunTestWithInputImagePacket(const Packet& input_image_packet, - cv::Mat expected_result, float range_min, - float range_max, int tensor_width, - int tensor_height, bool keep_aspect, - absl::optional border_mode, - const mediapipe::NormalizedRect& roi) { - std::string border_mode_str; - if (border_mode) { - switch (*border_mode) { - case BorderMode::kReplicate: - border_mode_str = "border_mode: BORDER_REPLICATE"; - break; - case BorderMode::kZero: - border_mode_str = "border_mode: BORDER_ZERO"; - break; - } - } - auto graph_config = mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "input_image" - input_stream: "roi" - node { - calculator: "ImageToTensorCalculator" - input_stream: "IMAGE:input_image" - input_stream: "NORM_RECT:roi" - output_stream: "TENSORS:tensor" - options { - [mediapipe.ImageToTensorCalculatorOptions.ext] { - output_tensor_width: $0 - output_tensor_height: $1 - keep_aspect_ratio: $4 - output_tensor_float_range { - min: $2 - max: $3 - } - $5 # border mode - } - } - } - )", - /*$0=*/tensor_width, - /*$1=*/tensor_height, - /*$2=*/range_min, - /*$3=*/range_max, - /*$4=*/keep_aspect ? "true" : "false", - /*$5=*/border_mode_str)); - - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - - MP_ASSERT_OK(graph.AddPacketToInputStream("input_image", input_image_packet)); - - MP_ASSERT_OK(graph.AddPacketToInputStream( - "roi", - MakePacket(std::move(roi)).At(Timestamp(0)))); - - MP_ASSERT_OK(graph.WaitUntilIdle()); - ASSERT_THAT(output_packets, testing::SizeIs(1)); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - ASSERT_THAT(tensor_vec, testing::SizeIs(1)); - - const Tensor& tensor = tensor_vec[0]; - EXPECT_EQ(tensor.element_type(), Tensor::ElementType::kFloat32); - - auto view = tensor.GetCpuReadView(); - cv::Mat tensor_mat(tensor_height, tensor_width, CV_32FC3, - const_cast(view.buffer())); - cv::Mat result_rgb; - auto transformation = - GetValueRangeTransformation(range_min, range_max, 0.0f, 255.0f).value(); - tensor_mat.convertTo(result_rgb, CV_8UC3, transformation.scale, - transformation.offset); - - cv::Mat diff; - cv::absdiff(result_rgb, expected_result, diff); - double max_val; - cv::minMaxLoc(diff, nullptr, &max_val); - // Expects the maximum absolute pixel-by-pixel difference is less than 5. - EXPECT_LE(max_val, 5); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.CloseInputStream("roi")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -Packet MakeImageFramePacket(cv::Mat input) { - ImageFrame input_image( - input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB, - input.cols, input.rows, input.step, input.data, [](uint8*) {}); - return MakePacket(std::move(input_image)).At(Timestamp(0)); -} - -Packet MakeImagePacket(cv::Mat input) { - mediapipe::Image input_image(std::make_shared( - input.channels() == 4 ? ImageFormat::SRGBA : ImageFormat::SRGB, - input.cols, input.rows, input.step, input.data, [](uint8*) {})); - return MakePacket(std::move(input_image)).At(Timestamp(0)); -} - -enum class InputType { kImageFrame, kImage }; - -const std::vector kInputTypesToTest = {InputType::kImageFrame, - InputType::kImage}; - -void RunTest(cv::Mat input, cv::Mat expected_result, float range_min, - float range_max, int tensor_width, int tensor_height, - bool keep_aspect, absl::optional border_mode, - const mediapipe::NormalizedRect& roi) { - for (auto input_type : kInputTypesToTest) { - RunTestWithInputImagePacket( - input_type == InputType::kImageFrame ? MakeImageFramePacket(input) - : MakeImagePacket(input), - expected_result, range_min, range_max, tensor_width, tensor_height, - keep_aspect, border_mode, roi); - } -} - -TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspect) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(0); - RunTest( - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, - /*border mode*/ {}, roi); -} - -TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(0); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "medium_sub_rect_keep_aspect_border_zero.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, - BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, MediumSubRectKeepAspectWithRotation) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(M_PI * 90.0f / 180.0f); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "medium_sub_rect_keep_aspect_with_rotation.png"), - /*range_min=*/0.0f, /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, - BorderMode::kReplicate, roi); -} - -TEST(ImageToTensorCalculatorTest, - MediumSubRectKeepAspectWithRotationBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(M_PI * 90.0f / 180.0f); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "medium_sub_rect_keep_aspect_with_rotation_border_zero.png"), - /*range_min=*/0.0f, /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/true, - BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotation) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(M_PI * -45.0f / 180.0f); - RunTest( - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb( - "/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png"), - /*range_min=*/-1.0f, - /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false, - BorderMode::kReplicate, roi); -} - -TEST(ImageToTensorCalculatorTest, MediumSubRectWithRotationBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.65f); - roi.set_y_center(0.4f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(M_PI * -45.0f / 180.0f); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "medium_sub_rect_with_rotation_border_zero.png"), - /*range_min=*/-1.0f, - /*range_max=*/1.0f, - /*tensor_width=*/256, /*tensor_height=*/256, /*keep_aspect=*/false, - BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, LargeSubRect) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(0); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/large_sub_rect.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false, - BorderMode::kReplicate, roi); -} - -TEST(ImageToTensorCalculatorTest, LargeSubRectBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(0); - RunTest( - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/false, - BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspect) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(0); - RunTest( - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, - BorderMode::kReplicate, roi); -} - -TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(0); - RunTest(GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "large_sub_rect_keep_aspect_border_zero.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, - BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, LargeSubRectKeepAspectWithRotation) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(M_PI * -15.0f / 180.0f); - RunTest(GetRgba("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "large_sub_rect_keep_aspect_with_rotation.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, - /*border_mode=*/{}, roi); -} - -TEST(ImageToTensorCalculatorTest, - LargeSubRectKeepAspectWithRotationBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.5f); - roi.set_height(1.1f); - roi.set_rotation(M_PI * -15.0f / 180.0f); - RunTest(GetRgba("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/" - "large_sub_rect_keep_aspect_with_rotation_border_zero.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/128, /*tensor_height=*/128, /*keep_aspect=*/true, - /*border_mode=*/BorderMode::kZero, roi); -} - -TEST(ImageToTensorCalculatorTest, NoOpExceptRange) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.0f); - roi.set_height(1.0f); - roi.set_rotation(0); - RunTest(GetRgba("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/noop_except_range.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true, - BorderMode::kReplicate, roi); -} - -TEST(ImageToTensorCalculatorTest, NoOpExceptRangeBorderZero) { - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.0f); - roi.set_height(1.0f); - roi.set_rotation(0); - RunTest(GetRgba("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/input.jpg"), - GetRgb("/mediapipe/calculators/" - "tensor/testdata/image_to_tensor/noop_except_range.png"), - /*range_min=*/0.0f, - /*range_max=*/1.0f, - /*tensor_width=*/64, /*tensor_height=*/128, /*keep_aspect=*/true, - BorderMode::kZero, roi); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter.h b/mediapipe/calculators/tensor/image_to_tensor_converter.h deleted file mode 100644 index 39fd1ee0d..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_ - -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -struct Size { - int width; - int height; -}; - -// Pixel extrapolation method. -// When converting image to tensor it may happen that tensor needs to read -// pixels outside image boundaries. Border mode helps to specify how such pixels -// will be calculated. -enum class BorderMode { kZero, kReplicate }; - -// Converts image to tensor. -class ImageToTensorConverter { - public: - virtual ~ImageToTensorConverter() = default; - - // Converts image to tensor. - // @image contains image to extract from. - // @roi describes region of interest within the image to extract (absolute - // values). - // @output_dims dimensions of output tensor. - // @range_min/max describes output tensor range image pixels should converted - // to. - virtual absl::StatusOr Convert(const mediapipe::Image& input, - const RotatedRect& roi, - const Size& output_dims, - float range_min, float range_max) = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc deleted file mode 100644 index 1c27f282a..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h" - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/common/types.h" -#include "tensorflow/lite/delegates/gpu/gl/command_queue.h" -#include "tensorflow/lite/delegates/gpu/gl/converters/util.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_call.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_texture.h" -#include "tensorflow/lite/delegates/gpu/gl/request_gpu_info.h" -#include "tensorflow/lite/delegates/gpu/gl/variable.h" - -namespace mediapipe { - -namespace { - -// Implements a common pattern of extracting a subrect from RGBA input texture -// and resizing it into a buffer. -class SubRectExtractorGl { - public: - // Extracts a region defined by @sub_rect, removes A channel, transforms input - // pixels as alpha * x + beta and resizes result into destination. - absl::Status ExtractSubRectToBuffer( - const tflite::gpu::gl::GlTexture& texture, - const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, - const tflite::gpu::HW& destination_size, - tflite::gpu::gl::CommandQueue* command_queue, - tflite::gpu::gl::GlBuffer* destination); - - static absl::StatusOr Create( - const mediapipe::GlContext& gl_context, bool input_starts_at_bottom, - BorderMode border_mode); - - private: - explicit SubRectExtractorGl(tflite::gpu::gl::GlProgram program, - tflite::gpu::uint3 workgroup_size, - bool use_custom_zero_border, - BorderMode border_mode) - : program_(std::move(program)), - workgroup_size_(workgroup_size), - use_custom_zero_border_(use_custom_zero_border), - border_mode_(border_mode) {} - - tflite::gpu::gl::GlProgram program_; - tflite::gpu::uint3 workgroup_size_; - bool use_custom_zero_border_ = false; - BorderMode border_mode_ = BorderMode::kReplicate; -}; - -absl::Status SetMat4x4(const tflite::gpu::gl::GlProgram& program, - const std::string& name, float* data) { - GLint uniform_id; - MP_RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetUniformLocation, &uniform_id, - program.id(), name.c_str())); - return TFLITE_GPU_CALL_GL(glProgramUniformMatrix4fv, program.id(), uniform_id, - 1, GL_TRUE, data); -} - -constexpr char kShaderCode[] = R"( -layout(std430) buffer; - -precision highp float; - -// It is possible to use "vec3 elements[];" here, however due to alignment -// requirements it works only when "packed" layout is used. "packed" layout is -// determined by implementation and it's expected that OpenGL API is used to -// query the layout. Favoring float array over vec3, considering performance is -// comparable, layout is the same and no need for layout querying (even though -// it's not quite needed here as there's only one member). -layout(binding = 0) writeonly buffer B0 { - float elements[]; -} output_data; - -uniform ivec2 out_size; -uniform float alpha; -uniform float beta; -uniform mat4 transform_matrix; -uniform mediump sampler2D input_data; - -void main() { - int out_width = out_size.x; - int out_height = out_size.y; - - ivec2 gid = ivec2(gl_GlobalInvocationID.xy); - if (gid.x >= out_width || gid.y >= out_height) { - return; - } - - // transform from image.width, image.height range to [0, 1] - float normal_x = (float(gid.x) + 0.5f) / float(out_width); - float normal_y = (float(gid.y) + 0.5f) / float(out_height); - vec4 tc = vec4(normal_x, normal_y, 0.0, 1.0); - - // Apply transformation from roi coordinates to original image coordinates. - tc = transform_matrix * tc; -#ifdef INPUT_STARTS_AT_BOTTOM - // Opengl texture sampler has origin in lower left corner, - // so we invert y coordinate. - tc.y = 1.0f - tc.y; -#endif // INPUT_STARTS_AT_BOTTOM - vec4 src_value = alpha * texture(input_data, tc.xy) + beta; - -#ifdef CUSTOM_ZERO_BORDER_MODE - float out_of_bounds = - float(tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0); - src_value = mix(src_value, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds); -#endif - - int linear_index = gid.y * out_width + gid.x; - - // output_data.elements is populated as though it contains vec3 elements. - int first_component_index = 3 * linear_index; - output_data.elements[first_component_index] = src_value.r; - output_data.elements[first_component_index + 1] = src_value.g; - output_data.elements[first_component_index + 2] = src_value.b; -} -)"; - -absl::Status SubRectExtractorGl::ExtractSubRectToBuffer( - const tflite::gpu::gl::GlTexture& texture, - const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect, - bool flip_horizontaly, float alpha, float beta, - const tflite::gpu::HW& destination_size, - tflite::gpu::gl::CommandQueue* command_queue, - tflite::gpu::gl::GlBuffer* destination) { - std::array transform_mat; - GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w, - texture_size.h, flip_horizontaly, - &transform_mat); - MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0)); - - // a) Filtering. - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - // b) Clamping. - switch (border_mode_) { - case BorderMode::kReplicate: { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - break; - } - case BorderMode::kZero: { - if (!use_custom_zero_border_) { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER); - glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, - std::array{0.0f, 0.0f, 0.0f, 0.0f}.data()); - } - break; - } - } - - MP_RETURN_IF_ERROR(destination->BindToIndex(0)); - MP_RETURN_IF_ERROR(program_.SetParameter({"input_data", 0})); - MP_RETURN_IF_ERROR( - SetMat4x4(program_, "transform_matrix", transform_mat.data())); - MP_RETURN_IF_ERROR(program_.SetParameter( - {"out_size", tflite::gpu::int2(destination_size.w, destination_size.h)})); - MP_RETURN_IF_ERROR(program_.SetParameter({"alpha", alpha})); - MP_RETURN_IF_ERROR(program_.SetParameter({"beta", beta})); - tflite::gpu::uint3 num_workgroups = tflite::gpu::DivideRoundUp( - tflite::gpu::uint3{destination_size.w, destination_size.h, 1}, - workgroup_size_); - MP_RETURN_IF_ERROR(command_queue->Dispatch(program_, num_workgroups)); - - // Resetting to MediaPipe texture param defaults. - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - - return absl::OkStatus(); -} - -absl::StatusOr SubRectExtractorGl::Create( - const mediapipe::GlContext& gl_context, bool input_starts_at_bottom, - BorderMode border_mode) { - bool use_custom_zero_border = border_mode == BorderMode::kZero && - !IsGlClampToBorderSupported(gl_context); - - const tflite::gpu::uint3 workgroup_size = {8, 8, 1}; - std::string starts_at_bottom_def; - if (input_starts_at_bottom) { - starts_at_bottom_def = R"( - #define INPUT_STARTS_AT_BOTTOM; - )"; - } - std::string custom_zero_border_mode_def; - if (use_custom_zero_border) { - custom_zero_border_mode_def = R"( - #define CUSTOM_ZERO_BORDER_MODE - )"; - } - const std::string full_shader_source = absl::StrCat( - tflite::gpu::gl::GetShaderHeader(workgroup_size), starts_at_bottom_def, - custom_zero_border_mode_def, kShaderCode); - - tflite::gpu::gl::GlShader shader; - MP_RETURN_IF_ERROR(tflite::gpu::gl::GlShader::CompileShader( - GL_COMPUTE_SHADER, full_shader_source, &shader)); - tflite::gpu::gl::GlProgram program; - MP_RETURN_IF_ERROR( - tflite::gpu::gl::GlProgram::CreateWithShader(shader, &program)); - - return SubRectExtractorGl(std::move(program), workgroup_size, - use_custom_zero_border, border_mode); -} - -class GlProcessor : public ImageToTensorConverter { - public: - absl::Status Init(CalculatorContext* cc, bool input_starts_at_bottom, - BorderMode border_mode) { - MP_RETURN_IF_ERROR(gl_helper_.Open(cc)); - return gl_helper_.RunInGlContext([this, input_starts_at_bottom, - border_mode]() -> absl::Status { - tflite::gpu::GpuInfo gpu_info; - MP_RETURN_IF_ERROR(tflite::gpu::gl::RequestGpuInfo(&gpu_info)); - RET_CHECK(gpu_info.IsApiOpenGl31OrAbove()) - << "OpenGL ES 3.1 is required."; - command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info); - - ASSIGN_OR_RETURN( - auto extractor, - SubRectExtractorGl::Create(gl_helper_.GetGlContext(), - input_starts_at_bottom, border_mode)); - extractor_ = absl::make_unique(std::move(extractor)); - return absl::OkStatus(); - }); - } - - absl::StatusOr Convert(const mediapipe::Image& input, - const RotatedRect& roi, - const Size& output_dims, float range_min, - float range_max) override { - if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) { - return InvalidArgumentError( - absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ", - static_cast(input.format()))); - } - - constexpr int kNumChannels = 3; - Tensor tensor(Tensor::ElementType::kFloat32, - {1, output_dims.height, output_dims.width, kNumChannels}); - - MP_RETURN_IF_ERROR(gl_helper_.RunInGlContext([this, &tensor, &input, &roi, - &output_dims, range_min, - range_max]() -> absl::Status { - constexpr int kRgbaNumChannels = 4; - auto source_texture = gl_helper_.CreateSourceTexture(input); - tflite::gpu::gl::GlTexture input_texture( - GL_TEXTURE_2D, source_texture.name(), GL_RGBA, - source_texture.width() * source_texture.height() * kRgbaNumChannels * - sizeof(uint8_t), - /*layer=*/0, - /*owned=*/false); - - constexpr float kInputImageRangeMin = 0.0f; - constexpr float kInputImageRangeMax = 1.0f; - ASSIGN_OR_RETURN( - auto transform, - GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax, - range_min, range_max)); - - auto buffer_view = tensor.GetOpenGlBufferWriteView(); - tflite::gpu::gl::GlBuffer output(GL_SHADER_STORAGE_BUFFER, - buffer_view.name(), tensor.bytes(), - /*offset=*/0, - /*has_ownership=*/false); - MP_RETURN_IF_ERROR(extractor_->ExtractSubRectToBuffer( - input_texture, - tflite::gpu::HW(source_texture.height(), source_texture.width()), roi, - /*flip_horizontaly=*/false, transform.scale, transform.offset, - tflite::gpu::HW(output_dims.height, output_dims.width), - command_queue_.get(), &output)); - - return absl::OkStatus(); - })); - - return std::move(tensor); - } - - ~GlProcessor() override { - gl_helper_.RunInGlContext([this]() { - // Release OpenGL resources. - extractor_ = nullptr; - command_queue_ = nullptr; - }); - } - - private: - std::unique_ptr command_queue_; - std::unique_ptr extractor_; - mediapipe::GlCalculatorHelper gl_helper_; -}; - -} // namespace - -absl::StatusOr> -CreateImageToGlBufferTensorConverter(CalculatorContext* cc, - bool input_starts_at_bottom, - BorderMode border_mode) { - auto result = absl::make_unique(); - MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode)); - - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h deleted file mode 100644 index 437b16b70..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_ - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -// Creates image to tensor (represented as OpenGL buffer) converter. -// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede -// converter creation. -absl::StatusOr> -CreateImageToGlBufferTensorConverter(CalculatorContext* cc, - bool input_starts_at_bottom, - BorderMode border_mode); - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_BUFFER_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc deleted file mode 100644 index 26c31eaf5..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h" - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" - -namespace mediapipe { - -namespace { - -constexpr int kAttribVertex = 0; -constexpr int kAttribTexturePosition = 1; -constexpr int kNumAttributes = 2; - -class GlProcessor : public ImageToTensorConverter { - public: - absl::Status Init(CalculatorContext* cc, bool input_starts_at_bottom, - BorderMode border_mode) { - MP_RETURN_IF_ERROR(gl_helper_.Open(cc)); - return gl_helper_.RunInGlContext([this, input_starts_at_bottom, - border_mode]() -> absl::Status { - use_custom_zero_border_ = - border_mode == BorderMode::kZero && - !IsGlClampToBorderSupported(gl_helper_.GetGlContext()); - border_mode_ = border_mode; - - const GLint attr_location[kNumAttributes] = { - kAttribVertex, - kAttribTexturePosition, - }; - const GLchar* attr_name[kNumAttributes] = { - "position", - "texture_coordinate", - }; - - constexpr GLchar kExtractSubRectVertexShader[] = R"( - in vec4 position; - in mediump vec4 texture_coordinate; - out mediump vec2 sample_coordinate; - uniform mat4 transform_matrix; - - void main() { - gl_Position = position; - // Apply transformation from roi coordinates to original image coordinates. - vec4 tc = transform_matrix * texture_coordinate; - #ifdef INPUT_STARTS_AT_BOTTOM - // Opengl texture sampler has origin in lower left corner, - // so we invert y coordinate. - tc.y = 1.0 - tc.y; - #endif // defined(INPUT_STARTS_AT_BOTTOM) - sample_coordinate = tc.xy; - } - )"; - - constexpr GLchar kExtractSubRectFragBody[] = R"( - DEFAULT_PRECISION(mediump, float) - - // Provided by kExtractSubRectVertexShader. - in vec2 sample_coordinate; - - uniform sampler2D input_texture; - uniform float alpha; - uniform float beta; - - #ifdef GL_ES - #define fragColor gl_FragColor - #else - out vec4 fragColor; - #endif // defined(GL_ES); - - void main() { - vec4 color = texture2D(input_texture, sample_coordinate); - #ifdef CUSTOM_ZERO_BORDER_MODE - float out_of_bounds = - float(sample_coordinate.x < 0.0 || sample_coordinate.x > 1.0 || - sample_coordinate.y < 0.0 || sample_coordinate.y > 1.0); - color = mix(color, vec4(0.0, 0.0, 0.0, 0.0), out_of_bounds); - #endif // defined(CUSTOM_ZERO_BORDER_MODE) - fragColor = alpha * color + beta; - } - )"; - - std::string starts_at_bottom_def; - if (input_starts_at_bottom) { - starts_at_bottom_def = R"( - #define INPUT_STARTS_AT_BOTTOM - )"; - } - - // Create program and set parameters. - const std::string extract_sub_rect_vertex_src = - absl::StrCat(mediapipe::kMediaPipeVertexShaderPreamble, - starts_at_bottom_def, kExtractSubRectVertexShader); - - std::string custom_zero_border_mode_def; - if (use_custom_zero_border_) { - custom_zero_border_mode_def = R"( - #define CUSTOM_ZERO_BORDER_MODE - )"; - } - const std::string extract_sub_rect_frag_src = - absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble, - custom_zero_border_mode_def, kExtractSubRectFragBody); - mediapipe::GlhCreateProgram(extract_sub_rect_vertex_src.c_str(), - extract_sub_rect_frag_src.c_str(), - kNumAttributes, &attr_name[0], attr_location, - &program_); - - RET_CHECK(program_) << "Problem initializing image to tensor program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_texture"), 1); - alpha_id_ = glGetUniformLocation(program_, "alpha"); - beta_id_ = glGetUniformLocation(program_, "beta"); - matrix_id_ = glGetUniformLocation(program_, "transform_matrix"); - - glGenFramebuffers(1, &framebuffer_); - - // vertex storage - glGenBuffers(2, vbo_); - glGenVertexArrays(1, &vao_); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]); - glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicSquareVertices), - mediapipe::kBasicSquareVertices, GL_STATIC_DRAW); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]); - glBufferData(GL_ARRAY_BUFFER, sizeof(mediapipe::kBasicTextureVertices), - mediapipe::kBasicTextureVertices, GL_STATIC_DRAW); - - glBindBuffer(GL_ARRAY_BUFFER, 0); - - return absl::OkStatus(); - }); - } - - absl::StatusOr Convert(const mediapipe::Image& input, - const RotatedRect& roi, - const Size& output_dims, float range_min, - float range_max) override { - if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) { - return InvalidArgumentError( - absl::StrCat("Only BGRA/RGBA textures are supported, passed format: ", - static_cast(input.format()))); - } - - constexpr int kNumChannels = 3; - Tensor tensor( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels}); - - MP_RETURN_IF_ERROR( - gl_helper_.RunInGlContext([this, &tensor, &input, &roi, &output_dims, - range_min, range_max]() -> absl::Status { - auto input_texture = gl_helper_.CreateSourceTexture(input); - - constexpr float kInputImageRangeMin = 0.0f; - constexpr float kInputImageRangeMax = 1.0f; - ASSIGN_OR_RETURN(auto transform, - GetValueRangeTransformation(kInputImageRangeMin, - kInputImageRangeMax, - range_min, range_max)); - auto tensor_view = tensor.GetOpenGlTexture2dWriteView(); - MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi, - /*flip_horizontaly=*/false, - transform.scale, transform.offset, - output_dims, &tensor_view)); - return absl::OkStatus(); - })); - - return tensor; - } - - absl::Status ExtractSubRect(const mediapipe::GlTexture& texture, - const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, - const Size& output_dims, - Tensor::OpenGlTexture2dView* output) { - std::array transform_mat; - - glDisable(GL_DEPTH_TEST); - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); - glViewport(0, 0, output_dims.width, output_dims.height); - - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, output->name()); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, - output->name(), 0); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(texture.target(), texture.name()); - - // a) Filtering. - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - // b) Clamping. - switch (border_mode_) { - case BorderMode::kReplicate: { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - break; - } - case BorderMode::kZero: { - if (!use_custom_zero_border_) { - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER); - glTexParameterfv(GL_TEXTURE_2D, GL_TEXTURE_BORDER_COLOR, - std::array{0.0f, 0.0f, 0.0f, 0.0f}.data()); - } - break; - } - } - - glUseProgram(program_); - glUniform1f(alpha_id_, alpha); - glUniform1f(beta_id_, beta); - - // If our context is ES2, then we must use GL_FALSE for our 'transpose' - // GLboolean in glUniformMatrix4fv, or else we'll get an INVALID_VALUE - // error. So in that case, we'll grab the transpose of our original matrix - // and send that instead. - const auto gl_context = mediapipe::GlContext::GetCurrent(); - LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; - if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) { - GetTransposedRotatedSubRectToRectTransformMatrix( - sub_rect, texture.width(), texture.height(), flip_horizontaly, - &transform_mat); - glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data()); - } else { - GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(), - texture.height(), flip_horizontaly, - &transform_mat); - glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data()); - } - - // vao - glBindVertexArray(vao_); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo_[0]); - glEnableVertexAttribArray(kAttribVertex); - glVertexAttribPointer(kAttribVertex, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo_[1]); - glEnableVertexAttribArray(kAttribTexturePosition); - glVertexAttribPointer(kAttribTexturePosition, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // Resetting to MediaPipe texture param defaults. - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - - glDisableVertexAttribArray(kAttribVertex); - glDisableVertexAttribArray(kAttribTexturePosition); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, 0); - - return absl::OkStatus(); - } - - ~GlProcessor() override { - gl_helper_.RunInGlContext([this]() { - // Release OpenGL resources. - if (framebuffer_ != 0) glDeleteFramebuffers(1, &framebuffer_); - if (program_ != 0) glDeleteProgram(program_); - if (vao_ != 0) glDeleteVertexArrays(1, &vao_); - glDeleteBuffers(2, vbo_); - }); - } - - private: - mediapipe::GlCalculatorHelper gl_helper_; - bool use_custom_zero_border_ = false; - BorderMode border_mode_ = BorderMode::kReplicate; - GLuint vao_ = 0; - GLuint vbo_[2] = {0, 0}; - GLuint program_ = 0; - GLuint framebuffer_ = 0; - GLint alpha_id_ = 0; - GLint beta_id_ = 0; - GLint matrix_id_ = 0; -}; - -} // namespace - -absl::StatusOr> -CreateImageToGlTextureTensorConverter(CalculatorContext* cc, - bool input_starts_at_bottom, - BorderMode border_mode) { - auto result = absl::make_unique(); - MP_RETURN_IF_ERROR(result->Init(cc, input_starts_at_bottom, border_mode)); - - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h deleted file mode 100644 index 269abf141..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_ - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -// Creates image to tensor (represented as OpenGL texture) converter. -// NOTE: mediapipe::GlCalculatorHelper::UpdateContract invocation must precede -// converter creation. -absl::StatusOr> -CreateImageToGlTextureTensorConverter(CalculatorContext* cc, - bool input_starts_at_bottom, - BorderMode border_mode); - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_TEXTURE_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc deleted file mode 100644 index 6fb39e0c3..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.cc +++ /dev/null @@ -1,88 +0,0 @@ -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#include -#include -#include - -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/gpu/gl_context.h" - -namespace mediapipe { - -namespace { - -class GlNoOpOverride : public GlOverride {}; - -class GlTexParameteriOverride : public GlOverride { - public: - GlTexParameteriOverride(GLenum name, GLint old_value) - : name_(name), old_value_(old_value) {} - - ~GlTexParameteriOverride() override { - glTexParameteri(GL_TEXTURE_2D, name_, old_value_); - } - - private: - GLenum name_; - GLint old_value_; -}; - -template -class GlTexParameterfvOverride : public GlOverride { - public: - GlTexParameterfvOverride(GLenum name, - std::array old_values) - : name_(name), old_values_(std::move(old_values)) {} - - ~GlTexParameterfvOverride() { - glTexParameterfv(GL_TEXTURE_2D, name_, &old_values_[0]); - } - - private: - GLenum name_; - std::array old_values_; -}; - -} // namespace - -std::unique_ptr OverrideGlTexParametri(GLenum name, GLint value) { - GLint old_value; - glGetTexParameteriv(GL_TEXTURE_2D, name, &old_value); - if (value != old_value) { - glTexParameteri(GL_TEXTURE_2D, name, value); - return {absl::make_unique(name, old_value)}; - } - return {absl::make_unique()}; -} - -template -std::unique_ptr OverrideGlTexParameterfv( - GLenum name, std::array values) { - std::array old_values; - glGetTexParameterfv(GL_TEXTURE_2D, name, values.data()); - if (values != old_values) { - glTexParameterfv(GL_TEXTURE_2D, name, values.data()); - return {absl::make_unique>( - name, std::move(old_values))}; - } - return {absl::make_unique()}; -} - -template std::unique_ptr OverrideGlTexParameterfv<4>( - GLenum name, std::array values); - -bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context) { - return gl_context.gl_major_version() > 3 || - (gl_context.gl_major_version() == 3 && - gl_context.gl_minor_version() >= 2); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h deleted file mode 100644 index 3105cfef1..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_ - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#include -#include -#include - -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/gpu/gl_context.h" - -namespace mediapipe { - -// Intended to override and automatically revert various OpenGL attributes. -// (e.g. overriding texture parameters like GL_TEXTURE_MIN_FILTER, -// GL_TEXTURE_MAG_FILTER, etc.) -class GlOverride { - public: - virtual ~GlOverride() = default; -}; - -// Creates an object that overrides attributes using `glTexParameteri` -// function during construction and reverts them during destruction. See -// `glTexParameteri` for details on @name and @value. -ABSL_MUST_USE_RESULT std::unique_ptr OverrideGlTexParametri( - GLenum name, GLint value); - -// Creates an object that overrides attributes using `glTexParameterfv` -// function during construction and reverts them during destruction. See -// `glTexParameterfv` for details on @name and @values. -template -ABSL_MUST_USE_RESULT std::unique_ptr OverrideGlTexParameterfv( - GLenum name, std::array values); - -bool IsGlClampToBorderSupported(const mediapipe::GlContext& gl_context); - -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_GL_UTILS_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc deleted file mode 100644 index 9482cfc2a..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils_test.cc +++ /dev/null @@ -1,49 +0,0 @@ -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 - -#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/gpu/gl_context.h" - -namespace mediapipe { -namespace { - -TEST(ImageToTensorConverterGlUtilsTest, GlTexParameteriOverrider) { - auto status_or_context = mediapipe::GlContext::Create(nullptr, false); - MP_ASSERT_OK(status_or_context); - auto context = status_or_context.value(); - - std::vector min_filter_changes; - context->Run([&min_filter_changes]() { - GLuint texture = 0; - glGenTextures(1, &texture); - glBindTexture(GL_TEXTURE_2D, texture); - - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - GLint value = 0; - glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value); - min_filter_changes.push_back(value); - - { - auto min_filter_linear = - OverrideGlTexParametri(GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value); - min_filter_changes.push_back(value); - - // reverter is destroyed automatically reverting previously set value - } - glGetTexParameteriv(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, &value); - min_filter_changes.push_back(value); - }); - - EXPECT_THAT(min_filter_changes, - testing::ElementsAre(GL_NEAREST, GL_LINEAR, GL_NEAREST)); -} - -} // namespace -} // namespace mediapipe - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_20 diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc deleted file mode 100644 index 1f86e1ced..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc +++ /dev/null @@ -1,408 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_converter_metal.h" - -#if MEDIAPIPE_METAL_ENABLED - -#import - -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/MPPMetalHelper.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/common/types.h" - -namespace mediapipe { - -namespace { - -// clang-format off -// a square formed by 2 triangles -const float kBasicSquareVertices[] = { - -1, 1, 0, 1, - 1, 1, 0, 1, - 1, -1, 0, 1, - -1, 1, 0, 1, - 1, -1, 0, 1, - -1, -1, 0, 1, -}; - -// maps a texture to kBasicSquareVertices via aspect fill -const float kBasicTextureVertices[] = { - 0, 0, 0, 1, - 1, 0, 0, 1, - 1, 1, 0, 1, - 0, 0, 0, 1, - 1, 1, 0, 1, - 0, 1, 0, 1, -}; -// clang-format on - -constexpr char kShaderLibHeader[] = R"( - #include - - using namespace metal; - - struct TextureVertex - { - float4 position [[position]]; - float2 uv; - }; -)"; - -constexpr char kVertexShader[] = R"( - vertex TextureVertex vertexShader( - constant float4 *position [[buffer(0)]], - device float4* tex_coords [[buffer(1)]], - constant float4x4& transform_matrix [[buffer(2)]], - uint vid [[vertex_id]]) { - TextureVertex vert; - vert.position = position[vid]; - vert.uv = (tex_coords[vid] * transform_matrix).xy; - return vert; - } -)"; - -constexpr char kFragmentShader[] = R"( - #ifdef OUTPUT_F16C4 - #define Type4 half4 - #define Type half - #endif // OUTPUT_F16C4 - - #ifdef OUTPUT_F32C4 - #define Type4 float4 - #define Type float - #endif // OUTPUT_F32C4 - - fragment Type4 fragmentShader(TextureVertex vertex_output [[stage_in]], - texture2d texture [[texture(0)]], - constant float* parameters [[buffer(1)]]) - { - const float alpha = parameters[0]; - const float beta = parameters[1]; - - #ifdef CLAMP_TO_ZERO - constexpr sampler linear_sampler(address::clamp_to_zero, min_filter::linear, - mag_filter::linear); - #endif // CLAMP_TO_ZERO - - #ifdef CLAMP_TO_EDGE - constexpr sampler linear_sampler(address::clamp_to_edge, min_filter::linear, - mag_filter::linear); - #endif // CLAMP_TO_EDGE - - Type4 texture_pixel = texture.sample(linear_sampler, vertex_output.uv); - return Type4(alpha * texture_pixel.rgb + beta, 0); - } -)"; - -enum class OutputFormat { kF16C4, kF32C4 }; - -MTLPixelFormat GetPixelFormat(OutputFormat output_format) { - switch (output_format) { - case OutputFormat::kF16C4: - return MTLPixelFormatRGBA16Float; - case OutputFormat::kF32C4: - return MTLPixelFormatRGBA32Float; - } -} -int GetBytesPerRaw(OutputFormat output_format, const tflite::gpu::HW& size) { - std::size_t type_size; - switch (output_format) { - case OutputFormat::kF16C4: - type_size = sizeof(tflite::gpu::HalfBits); - break; - case OutputFormat::kF32C4: - type_size = sizeof(float); - break; - } - constexpr int kNumChannels = 4; - return size.w * kNumChannels * type_size; -} - -class SubRectExtractorMetal { - public: - static absl::StatusOr> Make( - id device, OutputFormat output_format, - BorderMode border_mode) { - id pipeline_state; - MP_RETURN_IF_ERROR(SubRectExtractorMetal::MakePipelineState( - device, output_format, border_mode, &pipeline_state)); - - return absl::make_unique(device, pipeline_state, - output_format); - } - - SubRectExtractorMetal(id device, - id pipeline_state, - OutputFormat output_format) - : device_(device), - pipeline_state_(pipeline_state), - output_format_(output_format) { - positions_buffer_ = - [device_ newBufferWithBytes:kBasicSquareVertices - length:sizeof(kBasicSquareVertices) - options:MTLResourceOptionCPUCacheModeDefault]; - - tex_coords_buffer_ = - [device_ newBufferWithBytes:kBasicTextureVertices - length:sizeof(kBasicTextureVertices) - options:MTLResourceOptionCPUCacheModeDefault]; - } - - absl::Status Execute(id input_texture, - const RotatedRect& sub_rect, bool flip_horizontaly, - float alpha, float beta, - const tflite::gpu::HW& destination_size, - id command_buffer, - id destination) { - auto output_texture = MTLTextureWithBuffer(destination_size, destination); - return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha, - beta, destination_size, command_buffer, - output_texture); - } - - private: - id MTLTextureWithBuffer(const tflite::gpu::HW& size, - id buffer) { - MTLTextureDescriptor* texture_desc = [MTLTextureDescriptor - texture2DDescriptorWithPixelFormat:GetPixelFormat(output_format_) - width:size.w - height:size.h - mipmapped:NO]; - texture_desc.usage = MTLTextureUsageRenderTarget; - - NSUInteger output_bytes_per_row = GetBytesPerRaw(output_format_, size); - - id texture = - [buffer newTextureWithDescriptor:texture_desc - offset:0 - bytesPerRow:output_bytes_per_row]; - return texture; - } - - absl::Status InternalExecute(id input_texture, - const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, - const tflite::gpu::HW& destination_size, - id command_buffer, - id output_texture) { - RET_CHECK(command_buffer != nil); - RET_CHECK(output_texture != nil); - - // Obtain texture mapping coordinates transformation matrix and copy its - // data to the buffer. - std::array transform_mat; - GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width, - input_texture.height, - flip_horizontaly, &transform_mat); - id transform_mat_buffer = - [device_ newBufferWithBytes:&transform_mat - length:sizeof(transform_mat) - options:MTLResourceOptionCPUCacheModeDefault]; - - // Create parameters wrapper. - float parameters[] = {alpha, beta}; - - // Now everything is ready to go! - // Setup render pass. - MTLRenderPassDescriptor* render_pass_desc = - [MTLRenderPassDescriptor renderPassDescriptor]; - render_pass_desc.colorAttachments[0].texture = output_texture; - render_pass_desc.colorAttachments[0].storeAction = MTLStoreActionStore; - render_pass_desc.colorAttachments[0].loadAction = MTLLoadActionClear; - - // Setup render command encoder. - id command_encoder = - [command_buffer renderCommandEncoderWithDescriptor:render_pass_desc]; - [command_encoder setRenderPipelineState:pipeline_state_]; - [command_encoder setVertexBuffer:positions_buffer_ offset:0 atIndex:0]; - [command_encoder setVertexBuffer:tex_coords_buffer_ offset:0 atIndex:1]; - [command_encoder setVertexBuffer:transform_mat_buffer offset:0 atIndex:2]; - [command_encoder setFragmentTexture:input_texture atIndex:0]; - [command_encoder setFragmentBytes:¶meters - length:sizeof(parameters) - atIndex:1]; - - [command_encoder drawPrimitives:MTLPrimitiveTypeTriangle - vertexStart:0 - vertexCount:6]; - [command_encoder endEncoding]; - - return absl::OkStatus(); - } - - static absl::Status MakePipelineState( - id device, OutputFormat output_format, BorderMode border_mode, - id* pipeline_state) { - RET_CHECK(pipeline_state != nil); - - std::string output_type_def; - MTLPixelFormat pixel_format; - switch (output_format) { - case OutputFormat::kF16C4: - output_type_def = R"( - #define OUTPUT_F16C4 - )"; - break; - case OutputFormat::kF32C4: - output_type_def = R"( - #define OUTPUT_F32C4 - )"; - break; - } - - std::string clamp_def; - switch (border_mode) { - case BorderMode::kReplicate: { - clamp_def = R"( - #define CLAMP_TO_EDGE - )"; - break; - } - case BorderMode::kZero: { - clamp_def = R"( - #define CLAMP_TO_ZERO - )"; - break; - } - } - - std::string shader_lib = - absl::StrCat(kShaderLibHeader, output_type_def, clamp_def, - kVertexShader, kFragmentShader); - NSError* error = nil; - NSString* library_source = - [NSString stringWithUTF8String:shader_lib.c_str()]; - - id library = - [device newLibraryWithSource:library_source options:nil error:&error]; - RET_CHECK(library != nil) << "Couldn't create a shader library" - << [[error localizedDescription] UTF8String]; - - id vertex_function = - [library newFunctionWithName:@"vertexShader"]; - RET_CHECK(vertex_function != nil) - << "Failed creating a new vertex function!"; - - id fragment_function = - [library newFunctionWithName:@"fragmentShader"]; - RET_CHECK(fragment_function != nil) - << "Failed creating a new fragment function!"; - - MTLRenderPipelineDescriptor* pipelineDescriptor = - [MTLRenderPipelineDescriptor new]; - pipelineDescriptor.vertexFunction = vertex_function; - pipelineDescriptor.fragmentFunction = fragment_function; - pipelineDescriptor.colorAttachments[0].pixelFormat = - GetPixelFormat(output_format); - - *pipeline_state = - [device newRenderPipelineStateWithDescriptor:pipelineDescriptor - error:&error]; - RET_CHECK(error == nil) << "Couldn't create a pipeline state" - << [[error localizedDescription] UTF8String]; - - return absl::OkStatus(); - } - - id positions_buffer_; - id tex_coords_buffer_; - id device_; - id pipeline_state_; - OutputFormat output_format_; -}; - -class MetalProcessor : public ImageToTensorConverter { - public: - absl::Status Init(CalculatorContext* cc, BorderMode border_mode) { - metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(metal_helper_); - ASSIGN_OR_RETURN(extractor_, SubRectExtractorMetal::Make( - metal_helper_.mtlDevice, - OutputFormat::kF32C4, border_mode)); - return absl::OkStatus(); - } - - absl::StatusOr Convert(const mediapipe::Image& input, - const RotatedRect& roi, - const Size& output_dims, float range_min, - float range_max) override { - if (input.format() != mediapipe::GpuBufferFormat::kBGRA32) { - return InvalidArgumentError( - absl::StrCat("Only BGRA/RGBA textures are supported, passed " - "format: ", - static_cast(input.format()))); - } - - @autoreleasepool { - id texture = - [metal_helper_ metalTextureWithGpuBuffer:input.GetGpuBuffer()]; - - constexpr int kNumChannels = 4; - Tensor tensor(Tensor::ElementType::kFloat32, - Tensor::Shape{1, output_dims.height, output_dims.width, - kNumChannels}); - - constexpr float kInputImageRangeMin = 0.0f; - constexpr float kInputImageRangeMax = 1.0f; - ASSIGN_OR_RETURN( - auto transform, - GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax, - range_min, range_max)); - - id command_buffer = [metal_helper_ commandBuffer]; - const auto& buffer_view = tensor.GetMtlBufferWriteView(command_buffer); - MP_RETURN_IF_ERROR(extractor_->Execute( - texture, roi, - /*flip_horizontaly=*/false, transform.scale, transform.offset, - tflite::gpu::HW(output_dims.height, output_dims.width), - command_buffer, buffer_view.buffer())); - [command_buffer commit]; - return std::move(tensor); - } - } - - private: - MPPMetalHelper* metal_helper_ = nil; - std::unique_ptr extractor_; -}; - -} // namespace - -absl::StatusOr> CreateMetalConverter( - CalculatorContext* cc, BorderMode border_mode) { - auto result = absl::make_unique(); - MP_RETURN_IF_ERROR(result->Init(cc, border_mode)); - - // Simply "return std::move(result)" failed to build on macOS with bazel. - return std::unique_ptr(std::move(result)); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_METAL_ENABLED diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.h b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.h deleted file mode 100644 index 0fe5a87d0..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_ - -#include "mediapipe/framework/port.h" - -#if MEDIAPIPE_METAL_ENABLED - -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -// Creates Metal image-to-tensor converter. -// NOTE: [MPPMetalHelper updateContract:...] invocation must precede -// converter creation. -absl::StatusOr> CreateMetalConverter( - CalculatorContext* cc, BorderMode border_mode); - -} // namespace mediapipe - -#endif // MEDIAPIPE_METAL_ENABLED - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_METAL_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc deleted file mode 100644 index 04a4bbd97..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h" - -#include -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -namespace { - -class OpenCvProcessor : public ImageToTensorConverter { - public: - OpenCvProcessor(BorderMode border_mode) { - switch (border_mode) { - case BorderMode::kReplicate: - border_mode_ = cv::BORDER_REPLICATE; - break; - case BorderMode::kZero: - border_mode_ = cv::BORDER_CONSTANT; - break; - } - } - - absl::StatusOr Convert(const mediapipe::Image& input, - const RotatedRect& roi, - const Size& output_dims, float range_min, - float range_max) override { - if (input.image_format() != mediapipe::ImageFormat::SRGB && - input.image_format() != mediapipe::ImageFormat::SRGBA) { - return InvalidArgumentError( - absl::StrCat("Only RGBA/RGB formats are supported, passed format: ", - static_cast(input.image_format()))); - } - cv::Mat src = mediapipe::formats::MatView(&input); - - constexpr int kNumChannels = 3; - Tensor tensor( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, output_dims.height, output_dims.width, kNumChannels}); - auto buffer_view = tensor.GetCpuWriteView(); - cv::Mat dst(output_dims.height, output_dims.width, CV_32FC3, - buffer_view.buffer()); - - const cv::RotatedRect rotated_rect(cv::Point2f(roi.center_x, roi.center_y), - cv::Size2f(roi.width, roi.height), - roi.rotation * 180.f / M_PI); - cv::Mat src_points; - cv::boxPoints(rotated_rect, src_points); - - const float dst_width = output_dims.width; - const float dst_height = output_dims.height; - /* clang-format off */ - float dst_corners[8] = {0.0f, dst_height, - 0.0f, 0.0f, - dst_width, 0.0f, - dst_width, dst_height}; - /* clang-format on */ - - cv::Mat dst_points = cv::Mat(4, 2, CV_32F, dst_corners); - cv::Mat projection_matrix = - cv::getPerspectiveTransform(src_points, dst_points); - cv::Mat transformed; - cv::warpPerspective(src, transformed, projection_matrix, - cv::Size(dst_width, dst_height), - /*flags=*/cv::INTER_LINEAR, - /*borderMode=*/border_mode_); - - if (transformed.channels() > kNumChannels) { - cv::Mat proper_channels_mat; - cv::cvtColor(transformed, proper_channels_mat, cv::COLOR_RGBA2RGB); - transformed = proper_channels_mat; - } - - constexpr float kInputImageRangeMin = 0.0f; - constexpr float kInputImageRangeMax = 255.0f; - ASSIGN_OR_RETURN( - auto transform, - GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax, - range_min, range_max)); - transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset); - return std::move(tensor); - } - - private: - enum cv::BorderTypes border_mode_; -}; - -} // namespace - -absl::StatusOr> CreateOpenCvConverter( - CalculatorContext* cc, BorderMode border_mode) { - // Simply "return absl::make_unique()" failed to build on - // macOS with bazel. - return std::unique_ptr( - absl::make_unique(border_mode)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h deleted file mode 100644 index 3ccecc557..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_ - -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -// Creates OpenCV image-to-tensor converter. -absl::StatusOr> CreateOpenCvConverter( - CalculatorContext* cc, BorderMode border_mode); - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_utils.cc b/mediapipe/calculators/tensor/image_to_tensor_utils.cc deleted file mode 100644 index 6b3bf08cd..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_utils.cc +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" - -#include - -#include "absl/types/optional.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -RotatedRect GetRoi(int input_width, int input_height, - absl::optional norm_rect) { - if (norm_rect) { - return {/*center_x=*/norm_rect->x_center() * input_width, - /*center_y =*/norm_rect->y_center() * input_height, - /*width =*/norm_rect->width() * input_width, - /*height =*/norm_rect->height() * input_height, - /*rotation =*/norm_rect->rotation()}; - } - return {/*center_x=*/0.5f * input_width, - /*center_y =*/0.5f * input_height, - /*width =*/static_cast(input_width), - /*height =*/static_cast(input_height), - /*rotation =*/0}; -} - -absl::StatusOr> PadRoi(int input_tensor_width, - int input_tensor_height, - bool keep_aspect_ratio, - RotatedRect* roi) { - if (!keep_aspect_ratio) { - return std::array{0.0f, 0.0f, 0.0f, 0.0f}; - } - - RET_CHECK(input_tensor_width > 0 && input_tensor_height > 0) - << "Input tensor width and height must be > 0."; - const float tensor_aspect_ratio = - static_cast(input_tensor_height) / input_tensor_width; - - RET_CHECK(roi->width > 0 && roi->height > 0) - << "ROI width and height must be > 0."; - const float roi_aspect_ratio = roi->height / roi->width; - - float vertical_padding = 0.0f; - float horizontal_padding = 0.0f; - float new_width; - float new_height; - if (tensor_aspect_ratio > roi_aspect_ratio) { - new_width = roi->width; - new_height = roi->width * tensor_aspect_ratio; - vertical_padding = (1.0f - roi_aspect_ratio / tensor_aspect_ratio) / 2.0f; - } else { - new_width = roi->height / tensor_aspect_ratio; - new_height = roi->height; - horizontal_padding = (1.0f - tensor_aspect_ratio / roi_aspect_ratio) / 2.0f; - } - - roi->width = new_width; - roi->height = new_height; - - return std::array{horizontal_padding, vertical_padding, - horizontal_padding, vertical_padding}; -} - -absl::StatusOr GetValueRangeTransformation( - float from_range_min, float from_range_max, float to_range_min, - float to_range_max) { - RET_CHECK_LT(from_range_min, from_range_max) - << "Invalid FROM range: min >= max."; - RET_CHECK_LT(to_range_min, to_range_max) << "Invalid TO range: min >= max."; - const float scale = - (to_range_max - to_range_min) / (from_range_max - from_range_min); - const float offset = to_range_min - from_range_min * scale; - return ValueTransformation{scale, offset}; -} - -void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, - int rect_width, int rect_height, - bool flip_horizontaly, - std::array* matrix_ptr) { - std::array& matrix = *matrix_ptr; - // The resulting matrix is multiplication of below commented out matrices: - // post_scale_matrix - // * translate_matrix - // * rotate_matrix - // * flip_matrix - // * scale_matrix - // * initial_translate_matrix - - // Matrix to convert X,Y to [-0.5, 0.5] range "initial_translate_matrix" - // { 1.0f, 0.0f, 0.0f, -0.5f} - // { 0.0f, 1.0f, 0.0f, -0.5f} - // { 0.0f, 0.0f, 1.0f, 0.0f} - // { 0.0f, 0.0f, 0.0f, 1.0f} - - const float a = sub_rect.width; - const float b = sub_rect.height; - // Matrix to scale X,Y,Z to sub rect "scale_matrix" - // Z has the same scale as X. - // { a, 0.0f, 0.0f, 0.0f} - // {0.0f, b, 0.0f, 0.0f} - // {0.0f, 0.0f, a, 0.0f} - // {0.0f, 0.0f, 0.0f, 1.0f} - - const float flip = flip_horizontaly ? -1 : 1; - // Matrix for optional horizontal flip around middle of output image. - // { fl , 0.0f, 0.0f, 0.0f} - // { 0.0f, 1.0f, 0.0f, 0.0f} - // { 0.0f, 0.0f, 1.0f, 0.0f} - // { 0.0f, 0.0f, 0.0f, 1.0f} - - const float c = std::cos(sub_rect.rotation); - const float d = std::sin(sub_rect.rotation); - // Matrix to do rotation around Z axis "rotate_matrix" - // { c, -d, 0.0f, 0.0f} - // { d, c, 0.0f, 0.0f} - // { 0.0f, 0.0f, 1.0f, 0.0f} - // { 0.0f, 0.0f, 0.0f, 1.0f} - - const float e = sub_rect.center_x; - const float f = sub_rect.center_y; - // Matrix to do X,Y translation of sub rect within parent rect - // "translate_matrix" - // {1.0f, 0.0f, 0.0f, e } - // {0.0f, 1.0f, 0.0f, f } - // {0.0f, 0.0f, 1.0f, 0.0f} - // {0.0f, 0.0f, 0.0f, 1.0f} - - const float g = 1.0f / rect_width; - const float h = 1.0f / rect_height; - // Matrix to scale X,Y,Z to [0.0, 1.0] range "post_scale_matrix" - // {g, 0.0f, 0.0f, 0.0f} - // {0.0f, h, 0.0f, 0.0f} - // {0.0f, 0.0f, g, 0.0f} - // {0.0f, 0.0f, 0.0f, 1.0f} - - // row 1 - matrix[0] = a * c * flip * g; - matrix[1] = -b * d * g; - matrix[2] = 0.0f; - matrix[3] = (-0.5f * a * c * flip + 0.5f * b * d + e) * g; - - // row 2 - matrix[4] = a * d * flip * h; - matrix[5] = b * c * h; - matrix[6] = 0.0f; - matrix[7] = (-0.5f * b * c - 0.5f * a * d * flip + f) * h; - - // row 3 - matrix[8] = 0.0f; - matrix[9] = 0.0f; - matrix[10] = a * g; - matrix[11] = 0.0f; - - // row 4 - matrix[12] = 0.0f; - matrix[13] = 0.0f; - matrix[14] = 0.0f; - matrix[15] = 1.0f; -} - -void GetTransposedRotatedSubRectToRectTransformMatrix( - const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, std::array* matrix_ptr) { - std::array& matrix = *matrix_ptr; - // See comments in GetRotatedSubRectToRectTransformMatrix for detailed - // calculations. - const float a = sub_rect.width; - const float b = sub_rect.height; - const float flip = flip_horizontaly ? -1 : 1; - const float c = std::cos(sub_rect.rotation); - const float d = std::sin(sub_rect.rotation); - const float e = sub_rect.center_x; - const float f = sub_rect.center_y; - const float g = 1.0f / rect_width; - const float h = 1.0f / rect_height; - - // row 1 (indices 0,4,8,12 from non-transposed fcn) - matrix[0] = a * c * flip * g; - matrix[1] = a * d * flip * h; - matrix[2] = 0.0f; - matrix[3] = 0.0f; - - // row 2 (indices 1,5,9,13 from non-transposed fcn) - matrix[4] = -b * d * g; - matrix[5] = b * c * h; - matrix[6] = 0.0f; - matrix[7] = 0.0f; - - // row 3 (indices 2,6,10,14 from non-transposed fcn) - matrix[8] = 0.0f; - matrix[9] = 0.0f; - matrix[10] = a * g; - matrix[11] = 0.0f; - - // row 4 (indices 3,7,11,15 from non-transposed fcn) - matrix[12] = (-0.5f * a * c * flip + 0.5f * b * d + e) * g; - matrix[13] = (-0.5f * b * c - 0.5f * a * d * flip + f) * h; - matrix[14] = 0.0f; - matrix[15] = 1.0f; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/image_to_tensor_utils.h b/mediapipe/calculators/tensor/image_to_tensor_utils.h deleted file mode 100644 index f913875e3..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_utils.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_ - -#include - -#include "absl/types/optional.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { - -struct RotatedRect { - float center_x; - float center_y; - float width; - float height; - float rotation; -}; - -// Generates a new ROI or converts it from normalized rect. -RotatedRect GetRoi(int input_width, int input_height, - absl::optional norm_rect); - -// Pads ROI, so extraction happens correctly if aspect ratio is to be kept. -// Returns letterbox padding applied. -absl::StatusOr> PadRoi(int input_tensor_width, - int input_tensor_height, - bool keep_aspect_ratio, - RotatedRect* roi); - -// Represents a transformation of value which involves scaling and offsetting. -// To apply transformation: -// ValueTransformation transform = ... -// float transformed_value = transform.scale * value + transfrom.offset; -struct ValueTransformation { - float scale; - float offset; -}; - -// Returns value transformation to apply to a value in order to convert it from -// [from_range_min, from_range_max] into [to_range_min, to_range_max] range. -// from_range_min must be less than from_range_max -// to_range_min must be less than to_range_max -absl::StatusOr GetValueRangeTransformation( - float from_range_min, float from_range_max, float to_range_min, - float to_range_max); - -// Populates 4x4 "matrix" with row major order transformation matrix which -// maps (x, y) in range [0, 1] (describing points of @sub_rect) -// to (x', y') in range [0, 1]*** (describing points of a rect: -// [0, @rect_width] x [0, @rect_height] = RECT). -// -// *** (x', y') will go out of the range for points from @sub_rect -// which are not contained by RECT and it's expected behavior -// -// @sub_rect - rotated sub rect in absolute coordinates -// @rect_width - rect width -// @rect_height - rect height -// @flip_horizontaly - we need to flip the output buffer. -// @matrix - 4x4 matrix (array of 16 elements) to populate -void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, - int rect_width, int rect_height, - bool flip_horizontaly, - std::array* matrix); - -// Returns the transpose of the matrix found with -// "GetRotatedSubRectToRectTransformMatrix". That is to say, this populates a -// 4x4 "matrix" with col major order transformation matrix which maps (x, y) in -// range [0, 1] (describing points of @sub_rect) to (x', y') in range [0, 1]*** -// (describing points of a rect: [0, @rect_width] x [0, @rect_height] = RECT). -// -// *** (x', y') will go out of the range for points from @sub_rect -// which are not contained by RECT and it's expected behavior -// -// @sub_rect - rotated sub rect in absolute coordinates -// @rect_width - rect width -// @rect_height - rect height -// @flip_horizontaly - we need to flip the output buffer. -// @matrix - 4x4 matrix (array of 16 elements) to populate -void GetTransposedRotatedSubRectToRectTransformMatrix( - const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, std::array* matrix); - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_IMAGE_TO_TENSOR_UTILS_H_ diff --git a/mediapipe/calculators/tensor/image_to_tensor_utils_test.cc b/mediapipe/calculators/tensor/image_to_tensor_utils_test.cc deleted file mode 100644 index 814b4c34f..000000000 --- a/mediapipe/calculators/tensor/image_to_tensor_utils_test.cc +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" - -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -using ::testing::ElementsAre; -using ::testing::ElementsAreArray; - -testing::Matcher EqRotatedRect(float width, float height, - float center_x, float center_y, - float rotation) { - return testing::AllOf( - testing::Field(&RotatedRect::width, testing::FloatEq(width)), - testing::Field(&RotatedRect::height, testing::FloatEq(height)), - testing::Field(&RotatedRect::center_x, testing::FloatEq(center_x)), - testing::Field(&RotatedRect::center_y, testing::FloatEq(center_y)), - testing::Field(&RotatedRect::rotation, testing::FloatEq(rotation))); -} - -TEST(GetRoi, NoNormRect) { - EXPECT_THAT(GetRoi(4, 4, {}), EqRotatedRect(4, 4, 2, 2, 0)); - EXPECT_THAT(GetRoi(25, 15, {}), EqRotatedRect(25, 15, 12.5f, 7.5f, 0)); -} - -TEST(GetRoi, WholeImageNormRect) { - mediapipe::NormalizedRect norm_rect; - norm_rect.set_width(1.0f); - norm_rect.set_height(1.0f); - norm_rect.set_x_center(0.5f); - norm_rect.set_y_center(0.5f); - norm_rect.set_rotation(0.0f); - EXPECT_THAT(GetRoi(4, 4, norm_rect), EqRotatedRect(4, 4, 2, 2, 0)); - EXPECT_THAT(GetRoi(25, 15, norm_rect), EqRotatedRect(25, 15, 12.5f, 7.5f, 0)); -} - -TEST(GetRoi, ExpandedNormRect) { - mediapipe::NormalizedRect norm_rect; - norm_rect.set_width(4.0f); - norm_rect.set_height(2.0f); - norm_rect.set_x_center(0.5f); - norm_rect.set_y_center(1.0f); - norm_rect.set_rotation(3.0f); - EXPECT_THAT(GetRoi(4, 4, norm_rect), EqRotatedRect(16, 8, 2, 4, 3)); - EXPECT_THAT(GetRoi(25, 15, norm_rect), EqRotatedRect(100, 30, 12.5f, 15, 3)); -} - -TEST(PadRoi, NoPadding) { - RotatedRect roi{.center_x = 20, - .center_y = 10, - .width = 100, - .height = 200, - .rotation = 5}; - auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/false, &roi); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - ElementsAreArray({0.0f, 0.0f, 0.0f, 0.0f})); - EXPECT_THAT(roi, EqRotatedRect(100, 200, 20, 10, 5)); -} - -TEST(PadRoi, HorizontalPadding) { - RotatedRect roi{.center_x = 20, - .center_y = 10, - .width = 100, - .height = 200, - .rotation = 5}; - auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/true, &roi); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - ElementsAreArray({0.25f, 0.0f, 0.25f, 0.0f})); - EXPECT_THAT(roi, EqRotatedRect(200, 200, 20, 10, 5)); -} - -TEST(PadRoi, VerticalPadding) { - RotatedRect roi{ - .center_x = 1, .center_y = 2, .width = 21, .height = 19, .rotation = 3}; - const float expected_horizontal_padding = (21 - 19) / 2.0f / 21; - auto status_or_value = PadRoi(10, 10, /*keep_aspect_ratio=*/true, &roi); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT( - status_or_value.value(), - ElementsAre(testing::FloatEq(0.0f), - testing::FloatNear(expected_horizontal_padding, 1e-6), - testing::FloatEq(0.0f), - testing::FloatNear(expected_horizontal_padding, 1e-6))); - EXPECT_THAT(roi, EqRotatedRect(21, 21, 1, 2, 3)); -} - -testing::Matcher EqValueTransformation(float scale, - float offset) { - return ::testing::AllOf( - testing::Field(&ValueTransformation::scale, testing::FloatEq(scale)), - testing::Field(&ValueTransformation::offset, testing::FloatEq(offset))); -} - -TEST(GetValueRangeTransformation, PixelToFloatZeroCenter) { - auto status_or_value = GetValueRangeTransformation( - /*from_range_min=*/0.0f, /*from_range_max=*/255.0f, - /*to_range_min=*/-1.0f, /*to_range_max=*/1.0f); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - EqValueTransformation(/*scale=*/2 / 255.0f, - /*offset=*/-1.0f)); -} - -TEST(GetValueRangeTransformation, PixelToFloat) { - auto status_or_value = GetValueRangeTransformation( - /*from_range_min=*/0.0f, /*from_range_max=*/255.0f, - /*to_range_min=*/0.0f, /*to_range_max=*/1.0f); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - EqValueTransformation(/*scale=*/1 / 255.0f, - /*offset=*/0.0f)); -} - -TEST(GetValueRangeTransformation, FloatToFloatNoOp) { - auto status_or_value = GetValueRangeTransformation( - /*from_range_min=*/0.0f, /*from_range_max=*/1.0f, - /*to_range_min=*/0.0f, /*to_range_max=*/1.0f); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - EqValueTransformation(/*scale=*/1.0f, /*offset=*/0.0f)); -} - -TEST(GetValueRangeTransformation, PixelToPixelNoOp) { - auto status_or_value = GetValueRangeTransformation( - /*from_range_min=*/0.0f, /*from_range_max=*/255.0f, - /*to_range_min=*/0.0f, /*to_range_max=*/255.0f); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - EqValueTransformation(/*scale=*/1.0f, /*offset=*/0.0f)); -} - -TEST(GetValueRangeTransformation, FloatToPixel) { - auto status_or_value = GetValueRangeTransformation( - /*from_range_min=*/0.0f, /*from_range_max=*/1.0f, - /*to_range_min=*/0.0f, /*to_range_max=*/255.0f); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), - EqValueTransformation(/*scale=*/255.0f, /*offset=*/0.0f)); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator.cc b/mediapipe/calculators/tensor/inference_calculator.cc deleted file mode 100644 index 11256a338..000000000 --- a/mediapipe/calculators/tensor/inference_calculator.cc +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/inference_calculator.h" - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/string_view.h" -#include "mediapipe/framework/tool/subgraph_expansion.h" - -namespace mediapipe { -namespace api2 { - -class InferenceCalculatorSelectorImpl - : public SubgraphImpl { - public: - absl::StatusOr GetConfig( - const CalculatorGraphConfig::Node& subgraph_node) { - const auto& options = - Subgraph::GetOptions<::mediapipe::InferenceCalculatorOptions>( - subgraph_node); - std::vector impls; - const bool should_use_gpu = - !options.has_delegate() || // Use GPU delegate if not specified - (options.has_delegate() && options.delegate().has_gpu()); - if (should_use_gpu) { - impls.emplace_back("Metal"); - impls.emplace_back("Gl"); - } - impls.emplace_back("Cpu"); - for (const auto& suffix : impls) { - const auto impl = absl::StrCat("InferenceCalculator", suffix); - if (!mediapipe::CalculatorBaseRegistry::IsRegistered(impl)) continue; - CalculatorGraphConfig::Node impl_node = subgraph_node; - impl_node.set_calculator(impl); - return tool::MakeSingleNodeGraph(std::move(impl_node)); - } - return absl::UnimplementedError("no implementation available"); - } -}; - -absl::StatusOr> InferenceCalculator::GetModelAsPacket( - CalculatorContext* cc) { - const auto& options = cc->Options(); - if (!options.model_path().empty()) { - return TfLiteModelLoader::LoadFromPath(options.model_path()); - } - if (!kSideInModel(cc).IsEmpty()) return kSideInModel(cc); - return absl::Status(mediapipe::StatusCode::kNotFound, - "Must specify TFLite model as path or loaded model."); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator.h b/mediapipe/calculators/tensor/inference_calculator.h deleted file mode 100644 index 9fe06181c..000000000 --- a/mediapipe/calculators/tensor/inference_calculator.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_CALCULATOR_H_ - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/inference_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" - -namespace mediapipe { -namespace api2 { - -// Runs inference on the provided input Tensors and TFLite model. -// -// Creates an interpreter with given model and calls invoke(). -// Optionally run inference on CPU/GPU. -// -// This calculator can be used with TensorConverterCalculator to get the -// appropriate inputs. -// -// When the input tensors are on CPU, gpu inference is optional and can be -// specified in the calculator options. -// When the input tensors are on GPU, inference is GPU and output can be CPU or -// GPU. -// -// Input: -// TENSORS - Vector of Tensors -// -// Output: -// TENSORS - Vector of Tensors -// -// Input side packet: -// CUSTOM_OP_RESOLVER (optional) - Use a custom op resolver, -// instead of the builtin one. -// MODEL (optional) - Use to specify TfLite model -// (std::unique_ptr>) -// -// Example use: -// node { -// calculator: "InferenceCalculator" -// input_stream: "TENSORS:tensor_image" -// output_stream: "TENSORS:tensors" -// options: { -// [mediapipe.InferenceCalculatorOptions.ext] { -// model_path: "modelname.tflite" -// } -// } -// } -// -// or -// -// node { -// calculator: "InferenceCalculator" -// input_stream: "TENSORS:tensor_image" -// input_side_packet: "MODEL:model" -// output_stream: "TENSORS:tensors" -// options: { -// [mediapipe.InferenceCalculatorOptions.ext] { -// model_path: "modelname.tflite" -// delegate { gpu {} } -// } -// } -// } -// -// IMPORTANT Notes: -// Tensors are assumed to be ordered correctly (sequentially added to model). -// Input tensors are assumed to be of the correct size and already normalized. - -class InferenceCalculator : public NodeIntf { - public: - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr SideInput::Optional - kSideInCustomOpResolver{"CUSTOM_OP_RESOLVER"}; - static constexpr SideInput::Optional kSideInModel{"MODEL"}; - static constexpr Output> kOutTensors{"TENSORS"}; - MEDIAPIPE_NODE_CONTRACT(kInTensors, kSideInCustomOpResolver, kSideInModel, - kOutTensors); - - protected: - using TfLiteDelegatePtr = - std::unique_ptr>; - - absl::StatusOr> GetModelAsPacket( - CalculatorContext* cc); -}; - -struct InferenceCalculatorSelector : public InferenceCalculator { - static constexpr char kCalculatorName[] = "InferenceCalculator"; -}; - -struct InferenceCalculatorGl : public InferenceCalculator { - static constexpr char kCalculatorName[] = "InferenceCalculatorGl"; -}; - -struct InferenceCalculatorMetal : public InferenceCalculator { - static constexpr char kCalculatorName[] = "InferenceCalculatorMetal"; -}; - -struct InferenceCalculatorCpu : public InferenceCalculator { - static constexpr char kCalculatorName[] = "InferenceCalculatorCpu"; -}; - -} // namespace api2 -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_INFERENCE_CALCULATOR_H_ diff --git a/mediapipe/calculators/tensor/inference_calculator.proto b/mediapipe/calculators/tensor/inference_calculator.proto deleted file mode 100644 index e0b538a91..000000000 --- a/mediapipe/calculators/tensor/inference_calculator.proto +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "InferenceCalculator" -// input_stream: "TENSOR_IN:image_tensors" -// output_stream: "TENSOR_OUT:result_tensors" -// options { -// [mediapipe.InferenceCalculatorOptions.ext] { -// model_path: "model.tflite" -// delegate { gpu {} } -// } -// } -// } -// -message InferenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional InferenceCalculatorOptions ext = 336783863; - } - - message Delegate { - // Default inference provided by tflite. - message TfLite {} - // Delegate to run GPU inference depending on the device. - // (Can use OpenGl, OpenCl, Metal depending on the device.) - message Gpu { - // Experimental, Android/Linux only. Use TFLite GPU delegate API2 for - // the NN inference. - // example: - // delegate: { gpu { use_advanced_gpu_api: true } } - optional bool use_advanced_gpu_api = 1 [default = false]; - - // This option is valid for TFLite GPU delegate API2 only, - // Choose any of available APIs to force running inference using it. - enum Api { - ANY = 0; - OPENGL = 1; - OPENCL = 2; - } - optional Api api = 4 [default = ANY]; - - // This option is valid for TFLite GPU delegate API2 only, - // Set to true to use 16-bit float precision. If max precision is needed, - // set to false for 32-bit float calculations only. - optional bool allow_precision_loss = 3 [default = true]; - - // Load pre-compiled serialized binary cache to accelerate init process. - // Only available for OpenCL delegate on Android. - // Kernel caching will only be enabled if this path is set. - optional string cached_kernel_path = 2; - } - // Android only. - message Nnapi {} - message Xnnpack { - // Number of threads for XNNPACK delegate. (By default, calculator tries - // to choose optimal number of threads depending on the device.) - optional int32 num_threads = 1 [default = -1]; - } - - oneof delegate { - TfLite tflite = 1; - Gpu gpu = 2; - Nnapi nnapi = 3; - Xnnpack xnnpack = 4; - } - } - - // Path to the TF Lite model (ex: /path/to/modelname.tflite). - // On mobile, this is generally just modelname.tflite. - optional string model_path = 1; - - // Whether the TF Lite GPU or CPU backend should be used. Effective only when - // input tensors are on CPU. For input tensors on GPU, GPU backend is always - // used. - // DEPRECATED: configure "delegate" instead. - optional bool use_gpu = 2 [deprecated = true, default = false]; - - // Android only. When true, an NNAPI delegate will be used for inference. - // If NNAPI is not available, then the default CPU delegate will be used - // automatically. - // DEPRECATED: configure "delegate" instead. - optional bool use_nnapi = 3 [deprecated = true, default = false]; - - // The number of threads available to the interpreter. Effective only when - // input tensors are on CPU and 'use_gpu' is false. - optional int32 cpu_num_thread = 4 [default = -1]; - - // TfLite delegate to run inference. - // If not specified, TFLite GPU delegate is used by default (as if "gpu {}" - // is specified) unless GPU support is disabled in the build (i.e., with - // --define MEDIAPIPE_DISABLE_GPU=1), in which case regular TFLite on CPU is - // used (as if "tflite {}" is specified) except when building with emscripten - // where xnnpack is used. - // NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes - // precedence over use_* deprecated options.) - optional Delegate delegate = 5; -} diff --git a/mediapipe/calculators/tensor/inference_calculator_cpu.cc b/mediapipe/calculators/tensor/inference_calculator_cpu.cc deleted file mode 100644 index 0299ab526..000000000 --- a/mediapipe/calculators/tensor/inference_calculator_cpu.cc +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/inference_calculator.h" - -#if defined(MEDIAPIPE_ANDROID) -#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" -#endif // ANDROID - -#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__) -#include "mediapipe/util/cpu_util.h" -#endif // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__ - -#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h" - -namespace mediapipe { -namespace api2 { - -namespace { - -int GetXnnpackDefaultNumThreads() { -#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_IOS) || \ - defined(__EMSCRIPTEN_PTHREADS__) - constexpr int kMinNumThreadsByDefault = 1; - constexpr int kMaxNumThreadsByDefault = 4; - return std::clamp(NumCPUCores() / 2, kMinNumThreadsByDefault, - kMaxNumThreadsByDefault); -#else - return 1; -#endif // MEDIAPIPE_ANDROID || MEDIAPIPE_IOS || __EMSCRIPTEN_PTHREADS__ -} - -// Returns number of threads to configure XNNPACK delegate with. -// Returns user provided value if specified. Otherwise, tries to choose optimal -// number of threads depending on the device. -int GetXnnpackNumThreads(const mediapipe::InferenceCalculatorOptions& opts) { - static constexpr int kDefaultNumThreads = -1; - if (opts.has_delegate() && opts.delegate().has_xnnpack() && - opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) { - return opts.delegate().xnnpack().num_threads(); - } - return GetXnnpackDefaultNumThreads(); -} - -} // namespace - -class InferenceCalculatorCpuImpl - : public NodeImpl { - public: - static absl::Status UpdateContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadModel(CalculatorContext* cc); - absl::Status LoadDelegate(CalculatorContext* cc); - - // TfLite requires us to keep the model alive as long as the interpreter is. - Packet model_packet_; - std::unique_ptr interpreter_; - TfLiteDelegatePtr delegate_; -}; - -absl::Status InferenceCalculatorCpuImpl::UpdateContract( - CalculatorContract* cc) { - const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>(); - RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected()) - << "Either model as side packet or model path in options is required."; - - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorCpuImpl::Open(CalculatorContext* cc) { - MP_RETURN_IF_ERROR(LoadModel(cc)); - MP_RETURN_IF_ERROR(LoadDelegate(cc)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) { - if (kInTensors(cc).IsEmpty()) { - return absl::OkStatus(); - } - const auto& input_tensors = *kInTensors(cc); - RET_CHECK(!input_tensors.empty()); - auto output_tensors = absl::make_unique>(); - - // Read CPU input into tensors. - for (int i = 0; i < input_tensors.size(); ++i) { - const Tensor* input_tensor = &input_tensors[i]; - auto input_tensor_view = input_tensor->GetCpuReadView(); - auto input_tensor_buffer = input_tensor_view.buffer(); - float* local_tensor_buffer = interpreter_->typed_input_tensor(i); - std::memcpy(local_tensor_buffer, input_tensor_buffer, - input_tensor->bytes()); - } - - // Run inference. - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - - // Output result tensors (CPU). - const auto& tensor_indexes = interpreter_->outputs(); - output_tensors->reserve(tensor_indexes.size()); - for (int i = 0; i < tensor_indexes.size(); ++i) { - TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]); - output_tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{std::vector{ - tensor->dims->data, tensor->dims->data + tensor->dims->size}}); - auto cpu_view = output_tensors->back().GetCpuWriteView(); - std::memcpy(cpu_view.buffer(), tensor->data.f, - output_tensors->back().bytes()); - } - kOutTensors(cc).Send(std::move(output_tensors)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorCpuImpl::Close(CalculatorContext* cc) { - interpreter_ = nullptr; - delegate_ = nullptr; - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorCpuImpl::LoadModel(CalculatorContext* cc) { - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc)); - const auto& model = *model_packet_.Get(); - tflite::ops::builtin::BuiltinOpResolver op_resolver = - kSideInCustomOpResolver(cc).GetOr( - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()); - - tflite::InterpreterBuilder(model, op_resolver)(&interpreter_); - RET_CHECK(interpreter_); - -#if defined(__EMSCRIPTEN__) - interpreter_->SetNumThreads(1); -#else - interpreter_->SetNumThreads( - cc->Options().cpu_num_thread()); -#endif // __EMSCRIPTEN__ - - RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk); - // TODO: Support quantized tensors. - CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type != - kTfLiteAffineQuantization); - - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorCpuImpl::LoadDelegate(CalculatorContext* cc) { - const auto& calculator_opts = - cc->Options(); - if (calculator_opts.has_delegate() && - calculator_opts.delegate().has_tflite()) { - // Default tflite inference requeqsted - no need to modify graph. - return absl::OkStatus(); - } - -#if defined(MEDIAPIPE_ANDROID) - const bool nnapi_requested = calculator_opts.has_delegate() - ? calculator_opts.delegate().has_nnapi() - : calculator_opts.use_nnapi(); - if (nnapi_requested) { - // Attempt to use NNAPI. - // If not supported, the default CPU delegate will be created and used. - interpreter_->SetAllowFp16PrecisionForFp32(1); - delegate_ = TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) { - // No need to free according to tflite::NnApiDelegate() documentation. - }); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - return absl::OkStatus(); - } -#endif // MEDIAPIPE_ANDROID - -#if defined(__EMSCRIPTEN__) - const bool use_xnnpack = true; -#else - const bool use_xnnpack = calculator_opts.has_delegate() && - calculator_opts.delegate().has_xnnpack(); -#endif // defined(__EMSCRIPTEN__) - - if (use_xnnpack) { - TfLiteXNNPackDelegateOptions xnnpack_opts{}; - xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts); - delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts), - &TfLiteXNNPackDelegateDelete); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - } - - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc b/mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc deleted file mode 100644 index b6de30e3a..000000000 --- a/mediapipe/calculators/tensor/inference_calculator_face_detection_test.cc +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/memory/memory.h" -#include "absl/strings/match.h" -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" -#include "mediapipe/calculators/tensor/inference_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" -#include "mediapipe/framework/api2/builder.h" -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/graph_test_base.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/subgraph_expansion.h" -#include "mediapipe/framework/tool/test_util.h" - -namespace mediapipe { -namespace api2 { -namespace { - -using mediapipe::Detection; -using mediapipe::InferenceCalculatorOptions_Delegate; -using testing::ElementsAre; -using testing::EqualsProto; -using testing::proto::Approximately; - -struct Param { - std::string name; // Appended to the test name. - std::string impl_suffix; // Expected InferenceCalculator backend. - InferenceCalculatorOptions_Delegate delegate; -}; - -const std::vector& GetParams() { - static auto all_params = [] { - static std::vector p; - p.push_back({"TfLite", "Cpu"}); - p.back().delegate.mutable_tflite(); -#if TARGET_OS_IPHONE && !TARGET_IPHONE_SIMULATOR - // Metal is not available on the iOS simulator. - p.push_back({"Metal", "Metal"}); - p.back().delegate.mutable_gpu(); -#endif // TARGET_IPHONE_SIMULATOR -#if __ANDROID__ && 0 // Disabled for now since emulator can't go GLESv3 - p.push_back({"Gl", "Gl"}); - p.back().delegate.mutable_gpu(); - // This requires API level 27 - p.push_back({"NnApi", "Cpu"}); - p.back().delegate.mutable_nnapi(); -#endif // __ANDROID__ - p.push_back({"XnnPack", "Cpu"}); - p.back().delegate.mutable_xnnpack(); - return p; - }(); - return all_params; -} - -class InferenceCalculatorTest : public testing::TestWithParam { - protected: - void SetDelegateForParam(mediapipe::CalculatorGraphConfig_Node* node) { - *node->mutable_options() - ->MutableExtension(mediapipe::InferenceCalculatorOptions::ext) - ->mutable_delegate() = GetParam().delegate; - } -}; - -TEST_P(InferenceCalculatorTest, TestBackendSelection) { - CalculatorGraphConfig config; - auto node = config.add_node(); - node->set_calculator("InferenceCalculator"); - SetDelegateForParam(node); - MP_ASSERT_OK(tool::ExpandSubgraphs(&config)); - EXPECT_EQ(config.node(0).calculator(), - absl::StrCat("InferenceCalculator", GetParam().impl_suffix)); -} - -TEST_P(InferenceCalculatorTest, TestFaceDetection) { - CalculatorGraphConfig config; - ASSERT_TRUE(LoadTestGraph( - &config, file::JoinPath(GetTestRootDir(), - "mediapipe/calculators/tensor/" - "testdata/face_detection_test.binarypb"))); - - // Expand subgraphs to find any nested instances of InferenceCalculator. - MP_ASSERT_OK(tool::ExpandSubgraphs(&config)); - int found = 0; - for (auto& node : *config.mutable_node()) { - // The InferenceCalculator subgraph itself will have expanded to a specific - // implementation. Replace it. - // TODO: make it possible to exclude it from expansion above. - if (absl::StartsWith(node.calculator(), "InferenceCalculator")) { - ++found; - node.set_calculator("InferenceCalculator"); - SetDelegateForParam(&node); - } - } - ASSERT_EQ(found, 1); - - std::vector detection_packets; - tool::AddVectorSink("detections", &config, &detection_packets); - std::vector rendering_packets; - tool::AddVectorSink("rendering", &config, &rendering_packets); - - // Load test image. - std::unique_ptr input_image = LoadTestPng( - file::JoinPath(GetTestRootDir(), "mediapipe/objc/testdata/sergey.png")); - ASSERT_THAT(input_image, testing::NotNull()); - - std::unique_ptr expected_image = - LoadTestPng(file::JoinPath(GetTestRootDir(), - "mediapipe/calculators/tensor/" - "testdata/face_detection_expected.png")); - ASSERT_THAT(expected_image, testing::NotNull()); - - std::string binary; - Detection expected_detection; - MP_ASSERT_OK( - file::GetContents(file::JoinPath(GetTestRootDir(), - "mediapipe/calculators/tensor/" - "testdata/expected_detection.binarypb"), - &binary)); - expected_detection.ParseFromArray(binary.data(), binary.size()); - - // Prepare test inputs. - std::unordered_map> input_streams; - input_streams.insert(std::make_pair("image", std::move(input_image))); - std::string output_stream = "rendering"; - - // Test graph with relaxed color difference tolerance. - // Compare with CPU generated image. - Timestamp ts0 = Timestamp(0); - TestGraphConfig(config, input_streams, output_stream, expected_image, {}, ts0, - 2.0, 2.0, 1.0); - - ASSERT_EQ(detection_packets.size(), 1); - std::vector dets = - detection_packets[0].Get>(); -#if !defined(MEDIAPIPE_PROTO_LITE) - // Approximately is not available with lite protos (b/178137094). - EXPECT_THAT(dets, - ElementsAre(Approximately(EqualsProto(expected_detection)))); -#endif -} - -INSTANTIATE_TEST_SUITE_P(Implementation, InferenceCalculatorTest, - testing::ValuesIn(GetParams()), - [](const testing::TestParamInfo& info) { - return info.param.name; - }); - -} // namespace -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator_gl.cc b/mediapipe/calculators/tensor/inference_calculator_gl.cc deleted file mode 100644 index 5769df20e..000000000 --- a/mediapipe/calculators/tensor/inference_calculator_gl.cc +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/inference_calculator.h" -#include "mediapipe/util/tflite/config.h" - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/util/tflite/tflite_gpu_runner.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if defined(MEDIAPIPE_ANDROID) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/filesystem.h" -#include "mediapipe/util/android/file/base/helpers.h" -#endif // ANDROID - -namespace mediapipe { -namespace api2 { - -class InferenceCalculatorGlImpl - : public NodeImpl { - public: - static absl::Status UpdateContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status ReadKernelsFromFile(); - absl::Status WriteKernelsToFile(); - absl::Status LoadModel(CalculatorContext* cc); - absl::Status LoadDelegate(CalculatorContext* cc); - absl::Status InitTFLiteGPURunner(CalculatorContext* cc); - - // TfLite requires us to keep the model alive as long as the interpreter is. - Packet model_packet_; - std::unique_ptr interpreter_; - TfLiteDelegatePtr delegate_; - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - mediapipe::GlCalculatorHelper gpu_helper_; - std::unique_ptr tflite_gpu_runner_; - bool allow_precision_loss_ = false; - mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api - tflite_gpu_runner_api_; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - std::vector output_shapes_; - std::vector> gpu_buffers_in_; - std::vector> gpu_buffers_out_; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED - - bool use_advanced_gpu_api_ = false; - bool use_gpu_delegate_ = false; - - bool use_kernel_caching_ = false; - std::string cached_kernel_filename_; -}; - -absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) { - const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>(); - RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected()) - << "Either model as side packet or model path in options is required."; - - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) { - const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>(); - use_advanced_gpu_api_ = options.has_delegate() && - options.delegate().has_gpu() && - options.delegate().gpu().use_advanced_gpu_api(); - allow_precision_loss_ = options.delegate().gpu().allow_precision_loss(); - tflite_gpu_runner_api_ = options.delegate().gpu().api(); - use_kernel_caching_ = use_advanced_gpu_api_ && - options.delegate().gpu().has_cached_kernel_path(); - use_gpu_delegate_ = !use_advanced_gpu_api_; - - if (use_kernel_caching_) { -#ifdef MEDIAPIPE_ANDROID - cached_kernel_filename_ = options.delegate().gpu().cached_kernel_path() + - mediapipe::File::Basename(options.model_path()) + - ".ker"; -#endif // MEDIAPIPE_ANDROID - } - - // When use_advanced_gpu_api_, model loading is handled in InitTFLiteGPURunner - // for everything. - if (!use_advanced_gpu_api_) { - MP_RETURN_IF_ERROR(LoadModel(cc)); - } - - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, - &cc]() -> ::mediapipe::Status { - return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc) : LoadDelegate(cc); - })); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) { - if (kInTensors(cc).IsEmpty()) { - return absl::OkStatus(); - } - const auto& input_tensors = *kInTensors(cc); - RET_CHECK(!input_tensors.empty()); - auto output_tensors = absl::make_unique>(); - - if (use_advanced_gpu_api_) { - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext( - [this, &input_tensors, &output_tensors]() -> ::mediapipe::Status { - for (int i = 0; i < input_tensors.size(); ++i) { - MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor( - input_tensors[i].GetOpenGlBufferReadView().name(), i)); - } - output_tensors->reserve(output_shapes_.size()); - for (int i = 0; i < output_shapes_.size(); ++i) { - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - output_shapes_[i]); - MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor( - output_tensors->back().GetOpenGlBufferWriteView().name(), i)); - } - return absl::OkStatus(); - })); - } else { - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext( - [this, &input_tensors]() -> ::mediapipe::Status { - // Explicitly copy input. - for (int i = 0; i < input_tensors.size(); ++i) { - glBindBuffer(GL_COPY_READ_BUFFER, - input_tensors[i].GetOpenGlBufferReadView().name()); - glBindBuffer(GL_COPY_WRITE_BUFFER, - gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name()); - glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, - input_tensors[i].bytes()); - } - return absl::OkStatus(); - })); - } - - // Run inference. - if (use_advanced_gpu_api_) { - RET_CHECK(tflite_gpu_runner_->Invoke().ok()); - } else { - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - } - - if (use_gpu_delegate_) { - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext( - [this, &output_tensors]() -> ::mediapipe::Status { - output_tensors->reserve(output_shapes_.size()); - for (int i = 0; i < output_shapes_.size(); ++i) { - const auto& t = gpu_buffers_out_[i]; - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - gpu_buffers_out_[i]->shape()); - auto read_view = t->GetOpenGlBufferReadView(); - glBindBuffer(GL_COPY_READ_BUFFER, read_view.name()); - auto write_view = output_tensors->back().GetOpenGlBufferWriteView(); - glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name()); - glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, - t->bytes()); - } - return absl::OkStatus(); - })); - } - // Output tensors are already bound if use_advanced_gpu_api_ is true. - - kOutTensors(cc).Send(std::move(output_tensors)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::WriteKernelsToFile() { -#ifdef MEDIAPIPE_ANDROID - if (use_kernel_caching_) { - // Save kernel file. - auto kernel_cache = absl::make_unique>( - tflite_gpu_runner_->GetSerializedBinaryCache()); - std::string cache_str(kernel_cache->begin(), kernel_cache->end()); - MP_RETURN_IF_ERROR( - mediapipe::file::SetContents(cached_kernel_filename_, cache_str)); - } -#endif // MEDIAPIPE_ANDROID - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) { - MP_RETURN_IF_ERROR(WriteKernelsToFile()); - if (use_gpu_delegate_) { - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status { - gpu_buffers_in_.clear(); - gpu_buffers_out_.clear(); - return absl::OkStatus(); - })); - } - - interpreter_ = nullptr; - delegate_ = nullptr; - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::ReadKernelsFromFile() { -#ifdef MEDIAPIPE_ANDROID - if (use_kernel_caching_) { - // Load pre-compiled kernel file. - if (mediapipe::File::Exists(cached_kernel_filename_)) { - std::string cache_str; - MP_RETURN_IF_ERROR( - mediapipe::file::GetContents(cached_kernel_filename_, &cache_str)); - std::vector cache_vec(cache_str.begin(), cache_str.end()); - tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec)); - } - } -#endif // MEDIAPIPE_ANDROID - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::InitTFLiteGPURunner( - CalculatorContext* cc) { - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc)); - const auto& model = *model_packet_.Get(); - tflite::ops::builtin::BuiltinOpResolver op_resolver = - kSideInCustomOpResolver(cc).GetOr( - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()); - - // Create runner - tflite::gpu::InferenceOptions options; - options.priority1 = allow_precision_loss_ - ? tflite::gpu::InferencePriority::MIN_LATENCY - : tflite::gpu::InferencePriority::MAX_PRECISION; - options.priority2 = tflite::gpu::InferencePriority::AUTO; - options.priority3 = tflite::gpu::InferencePriority::AUTO; - options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; - tflite_gpu_runner_ = std::make_unique(options); - switch (tflite_gpu_runner_api_) { - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::OPENGL: { - tflite_gpu_runner_->ForceOpenGL(); - break; - } - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::OPENCL: { - tflite_gpu_runner_->ForceOpenCL(); - break; - } - case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: { - // Do not need to force any specific API. - break; - } - } - MP_RETURN_IF_ERROR(tflite_gpu_runner_->InitializeWithModel( - model, op_resolver, /*allow_quant_ops=*/true)); - - // Create and bind OpenGL buffers for outputs. - // The buffers are created once and their ids are passed to calculator outputs - output_shapes_.resize(tflite_gpu_runner_->outputs_size()); - for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) { - output_shapes_[i] = {tflite_gpu_runner_->GetOutputShapes()[i].b, - tflite_gpu_runner_->GetOutputShapes()[i].h, - tflite_gpu_runner_->GetOutputShapes()[i].w, - tflite_gpu_runner_->GetOutputShapes()[i].c}; - } - - MP_RETURN_IF_ERROR(ReadKernelsFromFile()); - - MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build()); - - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) { - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc)); - const auto& model = *model_packet_.Get(); - tflite::ops::builtin::BuiltinOpResolver op_resolver = - kSideInCustomOpResolver(cc).GetOr( - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()); - - tflite::InterpreterBuilder(model, op_resolver)(&interpreter_); - RET_CHECK(interpreter_); - -#if defined(__EMSCRIPTEN__) - interpreter_->SetNumThreads(1); -#else - interpreter_->SetNumThreads( - cc->Options().cpu_num_thread()); -#endif // __EMSCRIPTEN__ - - RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk); - // TODO: Support quantized tensors. - CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type != - kTfLiteAffineQuantization); - - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) { - // Configure and create the delegate. - TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault(); - options.compile_options.precision_loss_allowed = - allow_precision_loss_ ? 1 : 0; - options.compile_options.preferred_gl_object_type = - TFLITE_GL_OBJECT_TYPE_FASTEST; - options.compile_options.dynamic_batch_enabled = 0; - options.compile_options.inline_parameters = 1; - delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options), - &TfLiteGpuDelegateDelete); - - // Get input image sizes. - const auto& input_indices = interpreter_->inputs(); - for (int i = 0; i < input_indices.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]); - gpu_buffers_in_.emplace_back(absl::make_unique( - Tensor::ElementType::kFloat32, - Tensor::Shape{std::vector{ - tensor->dims->data, tensor->dims->data + tensor->dims->size}})); - RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( - delegate_.get(), - gpu_buffers_in_.back()->GetOpenGlBufferWriteView().name(), - interpreter_->inputs()[i]), - kTfLiteOk); - } - interpreter_->SetAllowBufferHandleOutput(true); - // Get output image sizes. - const auto& output_indices = interpreter_->outputs(); - output_shapes_.resize(output_indices.size()); - // Create and bind output buffers. - for (int i = 0; i < output_shapes_.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]); - gpu_buffers_out_.emplace_back(absl::make_unique( - Tensor::ElementType::kFloat32, - Tensor::Shape{std::vector{ - tensor->dims->data, tensor->dims->data + tensor->dims->size}})); - RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( - delegate_.get(), - gpu_buffers_out_.back()->GetOpenGlBufferWriteView().name(), - output_indices[i]), - kTfLiteOk); - } - - // Must call this last. - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator_metal.cc b/mediapipe/calculators/tensor/inference_calculator_metal.cc deleted file mode 100644 index 4bf3525e4..000000000 --- a/mediapipe/calculators/tensor/inference_calculator_metal.cc +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import -#import -#import - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/inference_calculator.h" -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/util/tflite/config.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h" - -namespace { - -// Round up n to next multiple of m. -template -T RoundUp(T n, T m) { - return ((n + m - T{1}) / m) * m; -} - -} // namespace - -namespace mediapipe { -namespace api2 { - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE -namespace { -tflite::gpu::BHWC BhwcFromTensorShape(const Tensor::Shape& shape) { - tflite::gpu::BHWC result; - result.b = shape.dims[0]; - switch (shape.dims.size()) { - case 1: - // result.b is already filled. - break; - case 2: - result.h = 1; - result.w = 1; - result.c = shape.dims[1]; - break; - case 3: - result.h = 1; - result.w = shape.dims[1]; - result.c = shape.dims[2]; - break; - case 4: - result.h = shape.dims[1]; - result.w = shape.dims[2]; - result.c = shape.dims[3]; - break; - default: - // Handles 0 and >4. - LOG(FATAL) - << "Dimensions size must be in range [1,4] for GPU inference, but " - << shape.dims.size() << " is provided"; - } - return result; -} -} // namespace -#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE - -class InferenceCalculatorMetalImpl - : public NodeImpl { - public: - static absl::Status UpdateContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadModel(CalculatorContext* cc); - absl::Status LoadDelegate(CalculatorContext* cc); - - // TfLite requires us to keep the model alive as long as the interpreter is. - Packet model_packet_; - std::unique_ptr interpreter_; - TfLiteDelegatePtr delegate_; - bool allow_precision_loss_ = false; - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE - MPPMetalHelper* gpu_helper_ = nullptr; - TFLBufferConvert* converter_to_BPHWC4_ = nil; - TFLBufferConvert* converter_from_BPHWC4_ = nil; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - std::vector output_shapes_; - std::vector> gpu_buffers_in_; - std::vector> gpu_buffers_out_; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED -}; - -absl::Status InferenceCalculatorMetalImpl::UpdateContract( - CalculatorContract* cc) { - const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>(); - RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected()) - << "Either model as side packet or model path in options is required."; - - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorMetalImpl::Open(CalculatorContext* cc) { - const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>(); - allow_precision_loss_ = options.delegate().gpu().allow_precision_loss(); - - MP_RETURN_IF_ERROR(LoadModel(cc)); - - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); - MP_RETURN_IF_ERROR(LoadDelegate(cc)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) { - if (kInTensors(cc).IsEmpty()) { - return absl::OkStatus(); - } - const auto& input_tensors = *kInTensors(cc); - RET_CHECK(!input_tensors.empty()); - auto output_tensors = absl::make_unique>(); - - id command_buffer; - - command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"InferenceCalculator"; - // Explicit copy input with conversion float 32 bits to 16 bits. - for (int i = 0; i < input_tensors.size(); ++i) { - auto input_view = input_tensors[i].GetMtlBufferReadView(command_buffer); - // Reshape tensor. - tflite::gpu::BHWC shape = BhwcFromTensorShape(input_tensors[i].shape()); - auto gpu_buffer_view = - gpu_buffers_in_[i]->GetMtlBufferWriteView(command_buffer); - id input_encoder = - [command_buffer computeCommandEncoder]; - [converter_to_BPHWC4_ convertWithEncoder:input_encoder - shape:shape - sourceBuffer:input_view.buffer() - convertedBuffer:gpu_buffer_view.buffer()]; - [input_encoder endEncoding]; - } - - // Run inference. - RET_CHECK(TFLGpuDelegateSetCommandBuffer(delegate_.get(), command_buffer)); - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - - output_tensors->reserve(output_shapes_.size()); - for (int i = 0; i < output_shapes_.size(); ++i) { - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - output_shapes_[i]); - // Reshape tensor. - tflite::gpu::BHWC shape = BhwcFromTensorShape(output_shapes_[i]); - auto read_view = gpu_buffers_out_[i]->GetMtlBufferReadView(command_buffer); - auto write_view = - output_tensors->at(i).GetMtlBufferWriteView(command_buffer); - id output_encoder = - [command_buffer computeCommandEncoder]; - [converter_from_BPHWC4_ convertWithEncoder:output_encoder - shape:shape - sourceBuffer:read_view.buffer() - convertedBuffer:write_view.buffer()]; - [output_encoder endEncoding]; - } - [command_buffer commit]; - - kOutTensors(cc).Send(std::move(output_tensors)); - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorMetalImpl::Close(CalculatorContext* cc) { - converter_to_BPHWC4_ = nil; - converter_from_BPHWC4_ = nil; - gpu_buffers_in_.clear(); - gpu_buffers_out_.clear(); - interpreter_ = nullptr; - delegate_ = nullptr; - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorMetalImpl::LoadModel(CalculatorContext* cc) { - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc)); - const auto& model = *model_packet_.Get(); - tflite::ops::builtin::BuiltinOpResolver op_resolver = - kSideInCustomOpResolver(cc).GetOr( - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates()); - - tflite::InterpreterBuilder(model, op_resolver)(&interpreter_); - RET_CHECK(interpreter_); - - interpreter_->SetNumThreads( - cc->Options().cpu_num_thread()); - - RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk); - // TODO: Support quantized tensors. - CHECK(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type != - kTfLiteAffineQuantization); - - return absl::OkStatus(); -} - -absl::Status InferenceCalculatorMetalImpl::LoadDelegate(CalculatorContext* cc) { - const auto& calculator_opts = - cc->Options(); - - // Configure and create the delegate. - TFLGpuDelegateOptions options; - // `enable_quantization` enables the run of sparse models i.e. the models with - // DENSIFY op preceding DEQUINTIZE op. Both ops get removed from the execution - // graph after the tensor of the weights is read. - options.enable_quantization = true; - options.allow_precision_loss = allow_precision_loss_; - options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypeDoNotWait; - delegate_ = - TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), &TFLGpuDelegateDelete); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - id device = gpu_helper_.mtlDevice; - - // Get input image sizes. - const auto& input_indices = interpreter_->inputs(); - for (int i = 0; i < input_indices.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]); - // Create and bind input buffer. - std::vector dims{tensor->dims->data, - tensor->dims->data + tensor->dims->size}; - dims.back() = RoundUp(dims.back(), 4); - gpu_buffers_in_.emplace_back(absl::make_unique( - allow_precision_loss_ ? Tensor::ElementType::kFloat16 - : Tensor::ElementType::kFloat32, - Tensor::Shape{dims})); - auto buffer_view = - gpu_buffers_in_[i]->GetMtlBufferWriteView(gpu_helper_.mtlDevice); - RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor( - delegate_.get(), input_indices[i], buffer_view.buffer()), - true); - } - - interpreter_->SetAllowBufferHandleOutput(true); - // Get output image sizes. - const auto& output_indices = interpreter_->outputs(); - output_shapes_.resize(output_indices.size()); - for (int i = 0; i < output_shapes_.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]); - RET_CHECK(tensor->dims->size <= 4); - // Create and bind output buffers. - // Channels are always padded to multiple of 4. - std::vector dims{tensor->dims->data, - tensor->dims->data + tensor->dims->size}; - output_shapes_[i] = {dims}; - dims.back() = RoundUp(dims.back(), 4); - gpu_buffers_out_.emplace_back(absl::make_unique( - allow_precision_loss_ ? Tensor::ElementType::kFloat16 - : Tensor::ElementType::kFloat32, - Tensor::Shape{dims})); - RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor( - delegate_.get(), output_indices[i], - gpu_buffers_out_[i] - ->GetMtlBufferWriteView(gpu_helper_.mtlDevice) - .buffer()), - true); - } - - // Create converter for GPU input. - converter_to_BPHWC4_ = - [[TFLBufferConvert alloc] initWithDevice:device - isFloat16:allow_precision_loss_ - convertToPBHWC4:true]; - if (converter_to_BPHWC4_ == nil) { - return mediapipe::InternalError( - "Error initializating input buffer converter"); - } - // Create converter for GPU output. - converter_from_BPHWC4_ = - [[TFLBufferConvert alloc] initWithDevice:device - isFloat16:allow_precision_loss_ - convertToPBHWC4:false]; - if (converter_from_BPHWC4_ == nil) { - return absl::InternalError("Error initializating output buffer converter"); - } - - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/inference_calculator_test.cc b/mediapipe/calculators/tensor/inference_calculator_test.cc deleted file mode 100644 index 882a5e81e..000000000 --- a/mediapipe/calculators/tensor/inference_calculator_test.cc +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/strings/str_replace.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/tensor/inference_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" - -#ifdef __APPLE__ -#include -#endif // defined(__APPLE__) - -namespace mediapipe { - -using ::tflite::Interpreter; - -void DoSmokeTest(const std::string& graph_proto) { - const int width = 8; - const int height = 8; - const int channels = 3; - // Prepare input tensor. - auto input_vec = absl::make_unique>(); - input_vec->emplace_back(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); - { - auto view1 = input_vec->back().GetCpuWriteView(); - auto tensor_buffer = view1.buffer(); - ASSERT_NE(tensor_buffer, nullptr); - for (int i = 0; i < width * height * channels - 1; i++) { - tensor_buffer[i] = 1; - } - } - - // Prepare single calculator graph to and wait for packets. - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(graph_proto); - std::vector output_packets; - tool::AddVectorSink("tensor_out", &graph_config, &output_packets); - CalculatorGraph graph(graph_config); - MP_ASSERT_OK(graph.StartRun({})); - - // Push the tensor into the graph. - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec.release()).At(Timestamp(0)))); - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - ASSERT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& result_vec = - output_packets[0].Get>(); - ASSERT_EQ(1, result_vec.size()); - - const Tensor& result = result_vec[0]; - auto view = result.GetCpuReadView(); - auto result_buffer = view.buffer(); - ASSERT_NE(result_buffer, nullptr); - for (int i = 0; i < width * height * channels - 1; i++) { - ASSERT_EQ(3, result_buffer[i]); - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -// Tests a simple add model that adds an input tensor to itself. -TEST(InferenceCalculatorTest, SmokeTest) { - std::string graph_proto = R"( - input_stream: "tensor_in" - node { - calculator: "InferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - options { - [mediapipe.InferenceCalculatorOptions.ext] { - model_path: "mediapipe/calculators/tensor/testdata/add.bin" - $delegate - } - } - } - )"; - // Test CPU inference only. - DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll( - graph_proto, {{"$delegate", "delegate { tflite {} }"}})); - DoSmokeTest(absl::StrReplaceAll(graph_proto, - {{"$delegate", "delegate { xnnpack {} }"}})); - DoSmokeTest(absl::StrReplaceAll( - graph_proto, - {{"$delegate", "delegate { xnnpack { num_threads: 10 } }"}})); -} - -TEST(InferenceCalculatorTest, SmokeTest_ModelAsInputSidePacket) { - std::string graph_proto = R"( - input_stream: "tensor_in" - - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:model_path" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { string_value: "mediapipe/calculators/tensor/testdata/add.bin" } - } - } - } - - node { - calculator: "LocalFileContentsCalculator" - input_side_packet: "FILE_PATH:model_path" - output_side_packet: "CONTENTS:model_blob" - } - - node { - calculator: "TfLiteModelCalculator" - input_side_packet: "MODEL_BLOB:model_blob" - output_side_packet: "MODEL:model" - } - - node { - calculator: "InferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - input_side_packet: "MODEL:model" - options { - [mediapipe.InferenceCalculatorOptions.ext] { - delegate { tflite {} } - } - } - } - )"; - DoSmokeTest(graph_proto); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc deleted file mode 100644 index 82180fe52..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ /dev/null @@ -1,671 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gpu_buffer.h" -#if MEDIAPIPE_METAL_ENABLED -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 -#include "mediapipe/gpu/gl_calculator_helper.h" -#if MEDIAPIPE_OPENGL_ES_VERSION < MEDIAPIPE_OPENGL_ES_31 -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#endif // MEDIAPIPE_OPENGL_ES_VERSION < MEDIAPIPE_OPENGL_ES_31 -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace { -constexpr int kWorkgroupSize = 8; // Block size for GPU shader. -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kMatrixTag[] = "MATRIX"; -} // namespace - -namespace mediapipe { - -// Calculator for normalizing and converting an ImageFrame, GpuBuffer or Matrix -// into a Tensor. -// -// This calculator is designed to be used with the TfLiteInferenceCalcualtor, -// as a pre-processing step for calculator inputs. -// -// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1], -// specified by options (unless outputting a quantized tensor). -// -// Input: -// One of the following tags: -// IMAGE - ImageFrame (assumed to be 8-bit or 32-bit data). -// IMAGE_GPU - GpuBuffer (assumed to be RGBA or RGB GL texture). -// MATRIX - Matrix. -// -// Output: -// One of the following tags: -// TENSORS - Vector of Tensors of type kFloat32. The resource type used: -// - MTLBuffer if Metal API is available -// - SSBO if Metal is unavailable and OpenGL ES 3.1 is available -// - Texture2D if Metal and GLES 3.1 are not available and GLES 3.0 is. -// -// Example use: -// node { -// calculator: "TensorConverterCalculator" -// input_stream: "IMAGE:input_image" -// output_stream: "TENSORS:image_tensor" -// options: { -// [mediapipe.TensorConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -// IMPORTANT Notes: -// GPU tensors are currently only supported on mobile platforms. - -class TensorConverterCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status InitGpu(CalculatorContext* cc); - absl::Status LoadOptions(CalculatorContext* cc); - template - absl::Status NormalizeImage(const ImageFrame& image_frame, - bool flip_vertically, float* tensor_ptr); - absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr); - absl::Status ProcessCPU(CalculatorContext* cc); - absl::Status ProcessGPU(CalculatorContext* cc); - -#if MEDIAPIPE_METAL_ENABLED - MPPMetalHelper* gpu_helper_ = nullptr; - id to_buffer_program_; -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - mediapipe::GlCalculatorHelper gpu_helper_; -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - GLuint to_buffer_program_; -#else - enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - GLuint to_tex2d_program_; - GLuint framebuffer_; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -#endif // MEDIAPIPE_METAL_ENABLED - - bool initialized_ = false; - bool use_gpu_ = false; - absl::optional> output_range_; - bool flip_vertically_ = false; - bool row_major_matrix_ = false; - int max_num_channels_ = 3; -}; -REGISTER_CALCULATOR(TensorConverterCalculator); - -absl::Status TensorConverterCalculator::GetContract(CalculatorContract* cc) { - // Confirm only one of the input streams is present. - RET_CHECK(static_cast(cc->Inputs().HasTag(kImageFrameTag)) + - static_cast(cc->Inputs().HasTag(kGpuBufferTag)) + - static_cast(cc->Inputs().HasTag(kMatrixTag)) == - 1); - - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } - if (cc->Inputs().HasTag(kMatrixTag)) { - cc->Inputs().Tag(kMatrixTag).Set(); - } - -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - cc->Inputs().Tag(kGpuBufferTag).Set(); -#if MEDIAPIPE_METAL_ENABLED - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // MEDIAPIPE_METAL_ENABLED - } -#endif // !MEDIAPIPE_DISABLE_GPU - - RET_CHECK(cc->Outputs().HasTag(kTensorsTag)); - cc->Outputs().Tag(kTensorsTag).Set>(); - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - use_gpu_ = true; -#if MEDIAPIPE_METAL_ENABLED - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // MEDIAPIPE_METAL_ENABLED - } -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::Process(CalculatorContext* cc) { - if (use_gpu_) { - if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) { - return absl::OkStatus(); - } - // Convert to GPU tensors type. - MP_RETURN_IF_ERROR(ProcessGPU(cc)); - } else { - // Convert to CPU tensors or Matrix type. - MP_RETURN_IF_ERROR(ProcessCPU(cc)); - } - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - if (use_gpu_) { -#if MEDIAPIPE_METAL_ENABLED - to_buffer_program_ = nil; -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - gpu_helper_.RunInGlContext([this] { -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - glDeleteProgram(to_buffer_program_); -#else - glDeleteFramebuffers(1, &framebuffer_); - glDeleteProgram(to_tex2d_program_); -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - }); -#endif // MEDIAPIPE_METAL_ENABLED - } -#endif // !MEDIAPIPE_DISABLE_GPU - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) { - auto output_tensors = absl::make_unique>(); - if (cc->Inputs().HasTag(kImageFrameTag)) { - if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto& image_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const mediapipe::ImageFormat::Format format = image_frame.Format(); - - if (!(format == mediapipe::ImageFormat::SRGBA || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::VEC32F1)) - RET_CHECK_FAIL() << "Unsupported CPU input format."; - - output_tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels_preserved}); - auto cpu_view = output_tensors->back().GetCpuWriteView(); - - // Copy image data into tensor. - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } - } else if (cc->Inputs().HasTag(kMatrixTag)) { - if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get(); - const int height = matrix.rows(); - const int width = matrix.cols(); - const int channels = 1; - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); - MP_RETURN_IF_ERROR(CopyMatrixToTensor( - matrix, output_tensors->back().GetCpuWriteView().buffer())); - } else { - return absl::OkStatus(); - } - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - if (!initialized_) { - MP_RETURN_IF_ERROR(InitGpu(cc)); - initialized_ = true; - } - const auto& input = - cc->Inputs().Tag(kGpuBufferTag).Get(); - int width = input.width(); - int height = input.height(); - int channels = max_num_channels_; - auto output_tensors = absl::make_unique>(); - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); -#if MEDIAPIPE_METAL_ENABLED - id device = gpu_helper_.mtlDevice; - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"TensorConverterCalculatorConvert"; - id compute_encoder = - [command_buffer computeCommandEncoder]; - [compute_encoder setComputePipelineState:to_buffer_program_]; - id src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input]; - [compute_encoder setTexture:src_texture atIndex:0]; - auto output_view = - output_tensors->at(0).GetMtlBufferWriteView(command_buffer); - [compute_encoder setBuffer:output_view.buffer() offset:0 atIndex:1]; - MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1); - MTLSize threadgroups = - MTLSizeMake(NumGroups(input.width(), kWorkgroupSize), - NumGroups(input.height(), kWorkgroupSize), 1); - [compute_encoder dispatchThreadgroups:threadgroups - threadsPerThreadgroup:threads_per_group]; - [compute_encoder endEncoding]; - [command_buffer commit]; -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext( - [this, &output_tensors, &input]() -> absl::Status { - auto src = gpu_helper_.CreateSourceTexture(input); -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - // Convert GL texture into SSBO. - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, src.name()); - auto output_view = output_tensors->back().GetOpenGlBufferWriteView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, output_view.name()); - glUseProgram(to_buffer_program_); - glDispatchCompute(NumGroups(input.width(), kWorkgroupSize), - NumGroups(input.height(), kWorkgroupSize), 1); - glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); - glBindTexture(GL_TEXTURE_2D, 0); -#else - // Texture2D -> Texture2D with OpenGL ES 3.0. - glUseProgram(to_tex2d_program_); - glDisable(GL_DEPTH_TEST); - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); - glViewport(0, 0, src.width(), src.height()); - glActiveTexture(GL_TEXTURE0); - auto output_view = output_tensors->back().GetOpenGlTexture2dWriteView(); - glBindTexture(GL_TEXTURE_2D, output_view.name()); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, - GL_TEXTURE_2D, output_view.name(), 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(src.target(), src.name()); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, - mediapipe::kBasicSquareVertices); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, - mediapipe::kBasicTextureVertices); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - src.Release(); - return absl::OkStatus(); - })); -#endif // MEDIAPIPE_METAL_ENABLED - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); -#else - RET_CHECK_FAIL() << "GPU processing is not enabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - // Get input image sizes. - const auto& input = - cc->Inputs().Tag(kGpuBufferTag).Get(); - mediapipe::ImageFormat::Format format = - mediapipe::ImageFormatForGpuBufferFormat(input.format()); - const bool include_alpha = (max_num_channels_ == 4); - const bool single_channel = (max_num_channels_ == 1); - if (!(format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Unsupported GPU input format."; - if (include_alpha && (format != mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Num input channels is less than desired output."; - -#if MEDIAPIPE_METAL_ENABLED - id device = gpu_helper_.mtlDevice; - // Shader to convert GL Texture to Metal Buffer, - // with normalization to either: [0,1] or [-1,1]. - const std::string shader_source = absl::Substitute( - R"( - #include - - using namespace metal; - - kernel void convertKernel( - texture2d in_tex [[ texture(0) ]], - device float* out_buf [[ buffer(1) ]], - uint2 gid [[ thread_position_in_grid ]]) { - if (gid.x >= in_tex.get_width() || gid.y >= in_tex.get_height()) return; - constexpr sampler texture_sampler(coord::pixel, address::clamp_to_edge); - const float2 coord = float2(gid.x, gid.y); - half4 pixel = in_tex.sample(texture_sampler, coord); - $0 // normalize [-1,1] - const int linear_index = $1 * ($2 * in_tex.get_width() + gid.x); - out_buf[linear_index + 0] = pixel.x; - $3 // g & b channels - $4 // alpha channel - } - )", - /*$0=*/ - output_range_.has_value() - ? absl::Substitute("pixel = pixel * half($0) + half($1);", - (output_range_->second - output_range_->first), - output_range_->first) - : "", - /*$1=*/max_num_channels_, - /*$2=*/flip_vertically_ ? "(in_tex.get_height() - 1 - gid.y)" : "gid.y", - /*$3=*/ - single_channel ? "" : R"(out_buf[linear_index + 1] = pixel.y; - out_buf[linear_index + 2] = pixel.z;)", - /*$4=*/include_alpha ? "out_buf[linear_index + 3] = pixel.w;" : ""); - - NSString* library_source = - [NSString stringWithUTF8String:shader_source.c_str()]; - NSError* error = nil; - id library = - [device newLibraryWithSource:library_source options:nullptr error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"convertKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - to_buffer_program_ = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(to_buffer_program_ != nil) << "Couldn't create pipeline state " << - [[error localizedDescription] UTF8String]; -#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &include_alpha, -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - &input, -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - &single_channel]() - -> absl::Status { -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - // Shader to convert GL Texture to Shader Storage Buffer Object (SSBO), - // with normalization to either: [0,1] or [-1,1]. - const std::string shader_source = absl::Substitute( - R"( #version 310 es - layout(local_size_x = $0, local_size_y = $0) in; - layout(binding = 0) uniform sampler2D input_texture; - layout(std430, binding = 1) buffer Output {float elements[];} output_data; - ivec2 width_height = ivec2($1, $2); - void main() { - ivec2 gid = ivec2(gl_GlobalInvocationID.xy); - if (gid.x >= width_height.x || gid.y >= width_height.y) return; - vec4 pixel = texelFetch(input_texture, gid, 0); - $3 // normalize [-1,1] - int linear_index = $7 * ($4 * width_height.x + gid.x); - output_data.elements[linear_index + 0] = pixel.x; // r channel - $5 // g & b channels - $6 // alpha channel - })", - /*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(), - /*$3=*/ - output_range_.has_value() - ? absl::Substitute("pixel = pixel * float($0) + float($1);", - (output_range_->second - output_range_->first), - output_range_->first) - : "", - /*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y", - /*$5=*/ - single_channel ? "" - : R"(output_data.elements[linear_index + 1] = pixel.y; - output_data.elements[linear_index + 2] = pixel.z;)", - /*$6=*/ - include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;" - : "", - /*$7=*/max_num_channels_); - GLuint shader = glCreateShader(GL_COMPUTE_SHADER); - const GLchar* sources[] = {shader_source.c_str()}; - glShaderSource(shader, 1, sources, NULL); - glCompileShader(shader); - GLint compiled = GL_FALSE; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); - RET_CHECK(compiled == GL_TRUE); - to_buffer_program_ = glCreateProgram(); - glAttachShader(to_buffer_program_, shader); - glDeleteShader(shader); - glLinkProgram(to_buffer_program_); -#else - // OpenGL ES 3.0 fragment shader Texture2d -> Texture2d conversion. - const std::string shader_source = absl::Substitute( - R"( - #if __VERSION__ < 130 - #define in varying - #endif // __VERSION__ < 130 - - #ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; - #else - #define lowp - #define mediump - #define highp - #define texture2D texture - out $0 fragColor; - #endif // defined(GL_ES) - - in vec2 sample_coordinate; - uniform sampler2D frame; - - void main() { - $1 // flip - vec4 pixel = texture2D(frame, sample_coordinate); - $2 // normalize [-1,1] - fragColor.r = pixel.r; // r channel - $3 // g & b channels - $4 // alpha channel - })", - /*$0=*/single_channel ? "vec1" : "vec4", - /*$1=*/ - flip_vertically_ ? "sample_coordinate.y = 1.0 - sample_coordinate.y;" - : "", - /*$2=*/output_range_.has_value() - ? absl::Substitute("pixel = pixel * float($0) + float($1);", - (output_range_->second - output_range_->first), - output_range_->first) - : "", - /*$3=*/single_channel ? "" : R"(fragColor.g = pixel.g; - fragColor.b = pixel.b;)", - /*$4=*/ - include_alpha ? "fragColor.a = pixel.a;" - : (single_channel ? "" : "fragColor.a = 1.0;")); - - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - // shader program and params - mediapipe::GlhCreateProgram( - mediapipe::kBasicVertexShader, shader_source.c_str(), NUM_ATTRIBUTES, - &attr_name[0], attr_location, &to_tex2d_program_); - RET_CHECK(to_tex2d_program_) << "Problem initializing the program."; - glUseProgram(to_tex2d_program_); - glUniform1i(glGetUniformLocation(to_tex2d_program_, "frame"), 1); - glGenFramebuffers(1, &framebuffer_); - -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - return absl::OkStatus(); - })); -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 -#endif // !MEDIAPIPE_DISABLE_GPU - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc) { - // Get calculator options specified in the graph. - const auto& options = - cc->Options<::mediapipe::TensorConverterCalculatorOptions>(); - - // if zero_center, set output float range to match [-1, 1] as specified in - // calculator proto. - if (options.zero_center()) { - output_range_.emplace(std::pair(-1.0, 1.0)); - } - - // Custom output_tensor_float_range values. - // If the float range is specified in pb text, use the specified values - // instead. - if (options.has_output_tensor_float_range()) { - output_range_.emplace(options.output_tensor_float_range().min(), - options.output_tensor_float_range().max()); - CHECK_GT(output_range_->second, output_range_->first); - } - - // Custom div and sub values. - if (options.use_custom_normalization()) { - output_range_.emplace(std::pair( - -options.custom_sub(), - -options.custom_sub() + 255.0 / options.custom_div())); - } - - // Get y-flip mode. - flip_vertically_ = options.flip_vertically(); - - // Get row_major_matrix mode. - row_major_matrix_ = options.row_major_matrix(); - - // Get desired way to handle input channels. - max_num_channels_ = options.max_num_channels(); - CHECK_GE(max_num_channels_, 1); - CHECK_LE(max_num_channels_, 4); - CHECK_NE(max_num_channels_, 2); - return absl::OkStatus(); -} - -template -absl::Status TensorConverterCalculator::NormalizeImage( - const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const int channels_ignored = channels - channels_preserved; - - if (output_range_.has_value()) { - // If the output float range is set and we are not using custom - // normalization, normalize the pixel values from [0, 255] to the specified - // output range. - RET_CHECK_NE(output_range_->first, output_range_->second); - const float scale = (output_range_->second - output_range_->first) / 255.0f; - const float bias = output_range_->first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - } else { - // [0,1], scale only (bias == 0) - // Verified that there are no precision issues with 1.0f / 255.0f expression - const float scale = 1.0f / 255.0f; - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale; - } - image_ptr += channels_ignored; - } - } - } - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix, - float* tensor_ptr) { - if (row_major_matrix_) { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.proto b/mediapipe/calculators/tensor/tensor_converter_calculator.proto deleted file mode 100644 index 97c2154a0..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.proto +++ /dev/null @@ -1,69 +0,0 @@ -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "TensorConverterCalculator" -// input_stream: "IMAGE_IN:input_image" -// output_stream: "TENSOR_OUT:image_tensor" -// options { -// [mediapipe.TensorConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -message TensorConverterCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorConverterCalculatorOptions ext = 335742637; - } - - // Choose normalization mode for output (not applied for Matrix inputs). - // true = [-1,1] - // false = [0,1] - // Ignored if using quantization. - optional bool zero_center = 1 [default = true]; - - // Custom settings to override the internal scaling factors `div` and `sub`. - // Both values must be set to non-negative values. Will only take effect on - // CPU AND when |use_custom_normalization| is set to true. When these custom - // values take effect, the |zero_center| setting above will be overriden, and - // the normalized_value will be calculated as: - // normalized_value = input / custom_div - custom_sub. - optional bool use_custom_normalization = 6 [default = false]; - optional float custom_div = 7 [default = -1.0]; - optional float custom_sub = 8 [default = -1.0]; - - // Whether the input image should be flipped vertically (along the - // y-direction). This is useful, for example, when the input image is defined - // with a coordinate system where the origin is at the bottom-left corner - // (e.g., in OpenGL) whereas the ML model expects an image with a top-left - // origin. - optional bool flip_vertically = 2 [default = false]; - - // Controls how many channels of the input image get passed through to the - // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU. - optional int32 max_num_channels = 3 [default = 3]; - - // The calculator expects Matrix inputs to be in column-major order. Set - // row_major_matrix to true if the inputs are in row-major order. - optional bool row_major_matrix = 4 [default = false]; - - // Quantization option (CPU only). - // When true, output kUint8 tensor instead of kFloat32. - optional bool use_quantized_tensors = 5 [default = false]; - - // Normalization option. - // Setting normalization_range results in the values normalized to - // the range [output_tensor_float_range.min, output_tensor_float_range.max]. - optional TensorFloatRange output_tensor_float_range = 9; - - message TensorFloatRange { - optional float min = 1; - optional float max = 2; - } -} diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc b/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc deleted file mode 100644 index bdea0795e..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/substitute.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { -namespace { - -constexpr char kTransposeOptionsString[] = - "[mediapipe.TensorConverterCalculatorOptions.ext]: {" - "row_major_matrix: True}"; - -} // namespace - -using RandomEngine = std::mt19937_64; -using testing::Eq; -const uint32 kSeed = 1234; -const int kNumSizes = 8; -const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, - {5, 3}, {7, 13}, {16, 32}, {101, 2}}; - -class TensorConverterCalculatorTest : public ::testing::Test { - protected: - // Adds a packet with a matrix filled with random values in [0,1]. - void AddRandomMatrix(int num_rows, int num_columns, uint32 seed, - bool row_major_matrix = false) { - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - auto matrix = ::absl::make_unique(); - matrix->resize(num_rows, num_columns); - if (row_major_matrix) { - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - float value = uniform_dist(random); - (*matrix)(y, x) = value; - } - } - } else { - for (int x = 0; x < num_columns; ++x) { - for (int y = 0; y < num_rows; ++y) { - float value = uniform_dist(random); - (*matrix)(y, x) = value; - } - } - } - MP_ASSERT_OK(graph_->AddPacketToInputStream( - "matrix", Adopt(matrix.release()).At(Timestamp(0)))); - } - - std::unique_ptr graph_; -}; - -TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "matrix" - node { - calculator: "TensorConverterCalculator" - input_stream: "MATRIX:matrix" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TensorConverterCalculatorOptions.ext] { - row_major_matrix: false - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - graph_ = absl::make_unique(); - MP_ASSERT_OK(graph_->Initialize(graph_config)); - MP_ASSERT_OK(graph_->StartRun({})); - - // Push the tensor into the graph. - AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/false); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const Tensor* tensor = &tensor_vec[0]; - EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - auto view = tensor->GetCpuReadView(); - auto tensor_buffer = view.buffer(); - for (int i = 0; i < num_rows * num_columns; ++i) { - const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph_->CloseInputStream("matrix")); - MP_ASSERT_OK(graph_->WaitUntilDone()); - - graph_.reset(); - } -} - -TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "matrix" - node { - calculator: "TensorConverterCalculator" - input_stream: "MATRIX:matrix" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TensorConverterCalculatorOptions.ext] { - row_major_matrix: true - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - graph_ = absl::make_unique(); - MP_ASSERT_OK(graph_->Initialize(graph_config)); - MP_ASSERT_OK(graph_->StartRun({})); - - // Push the tensor into the graph. - AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/true); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const Tensor* tensor = &tensor_vec[0]; - EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - auto view = tensor->GetCpuReadView(); - auto tensor_buffer = view.buffer(); - for (int i = 0; i < num_rows * num_columns; ++i) { - const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph_->CloseInputStream("matrix")); - MP_ASSERT_OK(graph_->WaitUntilDone()); - - graph_.reset(); - } -} - -TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) { - CalculatorGraph graph; - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "input_image" - node { - calculator: "TensorConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TensorConverterCalculatorOptions.ext] { - row_major_matrix: true - use_custom_normalization: true - custom_div: 2.0 - custom_sub: 33.0 - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const Tensor* tensor = &tensor_vec[0]; - EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); - auto view = tensor->GetCpuReadView(); - EXPECT_FLOAT_EQ(67.0f, *view.buffer()); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TensorConverterCalculatorTest, SetOutputRange) { - std::vector> range_values = { - std::make_pair(0.0, 1.0), std::make_pair(-1.0, 1.0), - std::make_pair(-0.5, 0.5)}; - for (std::pair range : range_values) { - CalculatorGraph graph; - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "input_image" - node { - calculator: "TensorConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TensorConverterCalculatorOptions.ext] { - output_tensor_float_range { - min: $0 - max: $1 - } - } - } - } - )", - /*$0=*/range.first, - /*$1=*/range.second)); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets.size(), Eq(1)); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_THAT(tensor_vec.size(), Eq(1)); - - const Tensor* tensor = &tensor_vec[0]; - - // Calculate the expected normalized value: - float normalized_value = - range.first + (200 * (range.second - range.first)) / 255.0; - - EXPECT_THAT(tensor->element_type(), Eq(Tensor::ElementType::kFloat32)); - auto view = tensor->GetCpuReadView(); - float dataf = *view.buffer(); - EXPECT_THAT( - normalized_value, - testing::FloatNear(dataf, 2.0f * std::abs(dataf) * - std::numeric_limits::epsilon())); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc b/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc deleted file mode 100644 index 87216f4d2..000000000 --- a/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/container/node_hash_map.h" -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { -namespace api2 { - -// Convert result tensors from classification models into MediaPipe -// classifications. -// -// Input: -// TENSORS - Vector of Tensors of type kFloat32 containing one -// tensor, the size of which must be (1, * num_classes). -// Output: -// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index -// fields of each classification are set, while the label -// field is only set if label_map_path is provided. -// -// Usage example: -// node { -// calculator: "TensorsToClassificationCalculator" -// input_stream: "TENSORS:tensors" -// output_stream: "CLASSIFICATIONS:classifications" -// options: { -// [mediapipe.TensorsToClassificationCalculatorOptions.ext] { -// num_classes: 1024 -// min_score_threshold: 0.1 -// label_map_path: "labelmap.txt" -// } -// } -// } -class TensorsToClassificationCalculator : public Node { - public: - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr Output kOutClassificationList{ - "CLASSIFICATIONS"}; - MEDIAPIPE_NODE_CONTRACT(kInTensors, kOutClassificationList); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - ::mediapipe::TensorsToClassificationCalculatorOptions options_; - int top_k_ = 0; - absl::node_hash_map label_map_; - bool label_map_loaded_ = false; -}; -MEDIAPIPE_REGISTER_NODE(TensorsToClassificationCalculator); - -absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) { - options_ = - cc->Options<::mediapipe::TensorsToClassificationCalculatorOptions>(); - - top_k_ = options_.top_k(); - if (options_.has_label_map_path()) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, - PathToResourceAsFile(options_.label_map_path())); - std::string label_map_string; - MP_RETURN_IF_ERROR( - mediapipe::GetResourceContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - label_map_loaded_ = true; - } else if (options_.has_label_map()) { - for (int i = 0; i < options_.label_map().entries_size(); ++i) { - const auto& entry = options_.label_map().entries(i); - RET_CHECK(!label_map_.contains(entry.id())) - << "Duplicate id found: " << entry.id(); - label_map_[entry.id()] = entry.label(); - } - label_map_loaded_ = true; - } - - return absl::OkStatus(); -} - -absl::Status TensorsToClassificationCalculator::Process(CalculatorContext* cc) { - const auto& input_tensors = *kInTensors(cc); - RET_CHECK_EQ(input_tensors.size(), 1); - - int num_classes = input_tensors[0].shape().num_elements(); - - if (options_.binary_classification()) { - RET_CHECK_EQ(num_classes, 1); - // Number of classes for binary classification. - num_classes = 2; - } - if (label_map_loaded_) { - RET_CHECK_EQ(num_classes, label_map_.size()); - } - auto view = input_tensors[0].GetCpuReadView(); - auto raw_scores = view.buffer(); - - auto classification_list = absl::make_unique(); - if (options_.binary_classification()) { - Classification* class_first = classification_list->add_classification(); - Classification* class_second = classification_list->add_classification(); - class_first->set_index(0); - class_second->set_index(1); - class_first->set_score(raw_scores[0]); - class_second->set_score(1. - raw_scores[0]); - - if (label_map_loaded_) { - class_first->set_label(label_map_[0]); - class_second->set_label(label_map_[1]); - } - } else { - for (int i = 0; i < num_classes; ++i) { - if (options_.has_min_score_threshold() && - raw_scores[i] < options_.min_score_threshold()) { - continue; - } - Classification* classification = - classification_list->add_classification(); - classification->set_index(i); - classification->set_score(raw_scores[i]); - - if (label_map_loaded_) { - classification->set_label(label_map_[i]); - } - } - } - - // Note that partial_sort will raise error when top_k_ > - // classification_list->classification_size(). - CHECK_GE(classification_list->classification_size(), top_k_); - auto raw_classification_list = classification_list->mutable_classification(); - if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { - std::partial_sort(raw_classification_list->begin(), - raw_classification_list->begin() + top_k_, - raw_classification_list->end(), - [](const Classification a, const Classification b) { - return a.score() > b.score(); - }); - - // Resizes the underlying list to have only top_k_ classifications. - raw_classification_list->DeleteSubrange( - top_k_, raw_classification_list->size() - top_k_); - } - kOutClassificationList(cc).Send(std::move(classification_list)); - return absl::OkStatus(); -} - -absl::Status TensorsToClassificationCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto b/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto deleted file mode 100644 index 3934a6101..000000000 --- a/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TensorsToClassificationCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorsToClassificationCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TensorsToClassificationCalculatorOptions ext = 335742638; - } - - message LabelMap { - message Entry { - optional int32 id = 1; - optional string label = 2; - } - repeated Entry entries = 1; - } - - // Score threshold for perserving the class. - optional float min_score_threshold = 1; - // Number of highest scoring labels to output. If top_k is not positive then - // all labels are used. - optional int32 top_k = 2; - // Path to a label map file for getting the actual name of class ids. - optional string label_map_path = 3; - // Label map. (Can be used instead of label_map_path.) - // NOTE: "label_map_path", if specified, takes precedence over "label_map". - optional LabelMap label_map = 5; - - // Whether the input is a single float for binary classification. - // When true, only a single float is expected in the input tensor and the - // label map, if provided, is expected to have exactly two labels. - // The single score(float) represent the probability of first label, and - // 1 - score is the probabilility of the second label. - optional bool binary_classification = 4; -} diff --git a/mediapipe/calculators/tensor/tensors_to_classification_calculator_test.cc b/mediapipe/calculators/tensor/tensors_to_classification_calculator_test.cc deleted file mode 100644 index 92b20629d..000000000 --- a/mediapipe/calculators/tensor/tensors_to_classification_calculator_test.cc +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -using mediapipe::ParseTextProtoOrDie; -using Node = ::mediapipe::CalculatorGraphConfig::Node; - -class TensorsToClassificationCalculatorTest : public ::testing::Test { - protected: - void BuildGraph(mediapipe::CalculatorRunner* runner, - const std::vector& scores) { - auto tensors = absl::make_unique>(); - tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, 1, static_cast(scores.size()), 1}); - auto view = tensors->back().GetCpuWriteView(); - float* tensor_buffer = view.buffer(); - ASSERT_NE(tensor_buffer, nullptr); - for (int i = 0; i < scores.size(); ++i) { - tensor_buffer[i] = scores[i]; - } - - int64 stream_timestamp = 0; - auto& input_stream_packets = - runner->MutableInputs()->Tag("TENSORS").packets; - - input_stream_packets.push_back( - mediapipe::Adopt(tensors.release()) - .At(mediapipe::Timestamp(stream_timestamp++))); - } -}; - -TEST_F(TensorsToClassificationCalculatorTest, CorrectOutput) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TensorsToClassificationCalculatorOptions.ext] {} - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label_id and score fields are set correctly. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_FALSE(classification_list.classification(i).has_label()); - } -} - -TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMapPath) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TensorsToClassificationCalculatorOptions.ext] { - label_map_path: "mediapipe/calculators/tensor/testdata/labelmap.txt" - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label field is set. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_TRUE(classification_list.classification(i).has_label()); - } -} - -TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithLabelMap) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TensorsToClassificationCalculatorOptions.ext] { - label_map { - entries { id: 0, label: "ClassA" } - entries { id: 1, label: "ClassB" } - entries { id: 2, label: "ClassC" } - } - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label field is set. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_TRUE(classification_list.classification(i).has_label()); - } -} - -TEST_F(TensorsToClassificationCalculatorTest, - CorrectOutputWithLabelMinScoreThreshold) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TensorsToClassificationCalculatorOptions.ext] { - min_score_threshold: 0.6 - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the low score labels are filtered out. - EXPECT_EQ(1, classification_list.classification_size()); - EXPECT_EQ(1, classification_list.classification(0).score()); -} - -TEST_F(TensorsToClassificationCalculatorTest, CorrectOutputWithTopK) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TensorsToClassificationCalculatorOptions.ext] { top_k: 2 } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the only top2 labels are left. - EXPECT_EQ(2, classification_list.classification_size()); - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ((classification_list.classification_size() - i) * 0.5, - classification_list.classification(i).score()); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc deleted file mode 100644 index f161127f5..000000000 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ /dev/null @@ -1,1163 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/formats/object_detection/anchor.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/ret_check.h" - -// Note: On Apple platforms MEDIAPIPE_DISABLE_GL_COMPUTE is automatically -// defined in mediapipe/framework/port.h. Therefore, -// "#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE" and "#if MEDIAPIPE_METAL_ENABLED" -// below are mutually exclusive. -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - -#if MEDIAPIPE_METAL_ENABLED -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#endif // MEDIAPIPE_METAL_ENABLED - -namespace { -constexpr int kNumInputTensorsWithAnchors = 3; -constexpr int kNumCoordsPerBox = 4; - -bool CanUseGpu() { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) || MEDIAPIPE_METAL_ENABLED -} -} // namespace - -namespace mediapipe { -namespace api2 { - -namespace { - -void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes, - std::vector* anchors) { - anchors->clear(); - for (int i = 0; i < num_boxes; ++i) { - Anchor new_anchor; - new_anchor.set_y_center(raw_anchors[i * kNumCoordsPerBox + 0]); - new_anchor.set_x_center(raw_anchors[i * kNumCoordsPerBox + 1]); - new_anchor.set_h(raw_anchors[i * kNumCoordsPerBox + 2]); - new_anchor.set_w(raw_anchors[i * kNumCoordsPerBox + 3]); - anchors->push_back(new_anchor); - } -} - -void ConvertAnchorsToRawValues(const std::vector& anchors, - int num_boxes, float* raw_anchors) { - CHECK_EQ(anchors.size(), num_boxes); - int box = 0; - for (const auto& anchor : anchors) { - raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); - raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center(); - raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h(); - raw_anchors[box * kNumCoordsPerBox + 3] = anchor.w(); - ++box; - } -} - -} // namespace - -// Convert result Tensors from object detection models into MediaPipe -// Detections. -// -// Input: -// TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have -// 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints. -// The size of the values must be (num_boxes * num_predicted_values). -// Second tensor is the score tensor. The size of the valuse must be -// (num_boxes * num_classes). It's optional to pass in a third tensor -// for anchors (e.g. for SSD models) depend on the outputs of the -// detection model. The size of anchor tensor must be (num_boxes * -// 4). -// -// Input side packet: -// ANCHORS (optional) - The anchors used for decoding the bounding boxes, as a -// vector of `Anchor` protos. Not required if post-processing is built-in -// the model. -// IGNORE_CLASSES (optional) - The list of class ids that should be ignored, as -// a vector of integers. It overrides the corresponding field in the -// calculator options. -// -// Output: -// DETECTIONS - Result MediaPipe detections. -// -// Usage example: -// node { -// calculator: "TensorsToDetectionsCalculator" -// input_stream: "TENSORS:tensors" -// input_side_packet: "ANCHORS:anchors" -// output_stream: "DETECTIONS:detections" -// options: { -// [mediapipe.TensorsToDetectionsCalculatorOptions.ext] { -// num_classes: 91 -// num_boxes: 1917 -// num_coords: 4 -// ignore_classes: [0, 1, 2] -// x_scale: 10.0 -// y_scale: 10.0 -// h_scale: 5.0 -// w_scale: 5.0 -// } -// } -// } -class TensorsToDetectionsCalculator : public Node { - public: - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr SideInput>::Optional kInAnchors{ - "ANCHORS"}; - static constexpr SideInput>::Optional kSideInIgnoreClasses{ - "IGNORE_CLASSES"}; - static constexpr Output> kOutDetections{"DETECTIONS"}; - MEDIAPIPE_NODE_CONTRACT(kInTensors, kInAnchors, kSideInIgnoreClasses, - kOutDetections); - static absl::Status UpdateContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status ProcessCPU(CalculatorContext* cc, - std::vector* output_detections); - absl::Status ProcessGPU(CalculatorContext* cc, - std::vector* output_detections); - - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status GpuInit(CalculatorContext* cc); - absl::Status DecodeBoxes(const float* raw_boxes, - const std::vector& anchors, - std::vector* boxes); - absl::Status ConvertToDetections(const float* detection_boxes, - const float* detection_scores, - const int* detection_classes, - std::vector* output_detections); - Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, - float box_xmax, float score, int class_id, - bool flip_vertically); - - int num_classes_ = 0; - int num_boxes_ = 0; - int num_coords_ = 0; - std::set ignore_classes_; - - ::mediapipe::TensorsToDetectionsCalculatorOptions options_; - std::vector anchors_; - -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint decode_program_; - GLuint score_program_; -#elif MEDIAPIPE_METAL_ENABLED - MPPMetalHelper* gpu_helper_ = nullptr; - id decode_program_; - id score_program_; -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - std::unique_ptr raw_anchors_buffer_; - std::unique_ptr decoded_boxes_buffer_; - std::unique_ptr scored_boxes_buffer_; - - bool gpu_inited_ = false; - bool gpu_input_ = false; - bool anchors_init_ = false; -}; -MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator); - -absl::Status TensorsToDetectionsCalculator::UpdateContract( - CalculatorContract* cc) { - if (CanUseGpu()) { -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#elif MEDIAPIPE_METAL_ENABLED - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - } - - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) { - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - if (CanUseGpu()) { -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#elif MEDIAPIPE_METAL_ENABLED - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - } - - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) { - auto output_detections = absl::make_unique>(); - bool gpu_processing = false; - if (CanUseGpu()) { - // Use GPU processing only if at least one input tensor is already on GPU - // (to avoid CPU->GPU overhead). - for (const auto& tensor : *kInTensors(cc)) { - if (tensor.ready_on_gpu()) { - gpu_processing = true; - break; - } - } - } - - if (gpu_processing) { - if (!gpu_inited_) { - MP_RETURN_IF_ERROR(GpuInit(cc)); - gpu_inited_ = true; - } - MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get())); - } else { - MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get())); - } - - kOutDetections(cc).Send(std::move(output_detections)); - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::ProcessCPU( - CalculatorContext* cc, std::vector* output_detections) { - const auto& input_tensors = *kInTensors(cc); - - if (input_tensors.size() == 2 || - input_tensors.size() == kNumInputTensorsWithAnchors) { - // Postprocessing on CPU for model without postprocessing op. E.g. output - // raw score tensor and box tensor. Anchor decoding will be handled below. - // TODO: Add flexible input tensor size handling. - auto raw_box_tensor = &input_tensors[0]; - RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3); - RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1); - RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_); - RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_); - auto raw_score_tensor = &input_tensors[1]; - RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3); - RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1); - RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_); - RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_); - auto raw_box_view = raw_box_tensor->GetCpuReadView(); - auto raw_boxes = raw_box_view.buffer(); - auto raw_scores_view = raw_score_tensor->GetCpuReadView(); - auto raw_scores = raw_scores_view.buffer(); - - // TODO: Support other options to load anchors. - if (!anchors_init_) { - if (input_tensors.size() == kNumInputTensorsWithAnchors) { - auto anchor_tensor = &input_tensors[2]; - RET_CHECK_EQ(anchor_tensor->shape().dims.size(), 2); - RET_CHECK_EQ(anchor_tensor->shape().dims[0], num_boxes_); - RET_CHECK_EQ(anchor_tensor->shape().dims[1], kNumCoordsPerBox); - auto anchor_view = anchor_tensor->GetCpuReadView(); - auto raw_anchors = anchor_view.buffer(); - ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_); - } else if (!kInAnchors(cc).IsEmpty()) { - anchors_ = *kInAnchors(cc); - } else { - return absl::UnavailableError("No anchor data available."); - } - anchors_init_ = true; - } - std::vector boxes(num_boxes_ * num_coords_); - MP_RETURN_IF_ERROR(DecodeBoxes(raw_boxes, anchors_, &boxes)); - - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - - // Filter classes by scores. - for (int i = 0; i < num_boxes_; ++i) { - int class_id = -1; - float max_score = -std::numeric_limits::max(); - // Find the top score for box i. - for (int score_idx = 0; score_idx < num_classes_; ++score_idx) { - if (ignore_classes_.find(score_idx) == ignore_classes_.end()) { - auto score = raw_scores[i * num_classes_ + score_idx]; - if (options_.sigmoid_score()) { - if (options_.has_score_clipping_thresh()) { - score = score < -options_.score_clipping_thresh() - ? -options_.score_clipping_thresh() - : score; - score = score > options_.score_clipping_thresh() - ? options_.score_clipping_thresh() - : score; - } - score = 1.0f / (1.0f + std::exp(-score)); - } - if (max_score < score) { - max_score = score; - class_id = score_idx; - } - } - } - detection_scores[i] = max_score; - detection_classes[i] = class_id; - } - - MP_RETURN_IF_ERROR( - ConvertToDetections(boxes.data(), detection_scores.data(), - detection_classes.data(), output_detections)); - } else { - // Postprocessing on CPU with postprocessing op (e.g. anchor decoding and - // non-maximum suppression) within the model. - RET_CHECK_EQ(input_tensors.size(), 4); - - auto num_boxes_tensor = &input_tensors[3]; - RET_CHECK_EQ(num_boxes_tensor->shape().dims.size(), 1); - RET_CHECK_EQ(num_boxes_tensor->shape().dims[0], 1); - - auto detection_boxes_tensor = &input_tensors[0]; - RET_CHECK_EQ(detection_boxes_tensor->shape().dims.size(), 3); - RET_CHECK_EQ(detection_boxes_tensor->shape().dims[0], 1); - const int max_detections = detection_boxes_tensor->shape().dims[1]; - RET_CHECK_EQ(detection_boxes_tensor->shape().dims[2], num_coords_); - - auto detection_classes_tensor = &input_tensors[1]; - RET_CHECK_EQ(detection_classes_tensor->shape().dims.size(), 2); - RET_CHECK_EQ(detection_classes_tensor->shape().dims[0], 1); - RET_CHECK_EQ(detection_classes_tensor->shape().dims[1], max_detections); - - auto detection_scores_tensor = &input_tensors[2]; - RET_CHECK_EQ(detection_scores_tensor->shape().dims.size(), 2); - RET_CHECK_EQ(detection_scores_tensor->shape().dims[0], 1); - RET_CHECK_EQ(detection_scores_tensor->shape().dims[1], max_detections); - - auto num_boxes_view = num_boxes_tensor->GetCpuReadView(); - auto num_boxes = num_boxes_view.buffer(); - num_boxes_ = num_boxes[0]; - - auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView(); - auto detection_boxes = detection_boxes_view.buffer(); - - auto detection_scores_view = detection_scores_tensor->GetCpuReadView(); - auto detection_scores = detection_scores_view.buffer(); - - auto detection_classes_view = detection_classes_tensor->GetCpuReadView(); - auto detection_classes_ptr = detection_classes_view.buffer(); - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { - detection_classes[i] = static_cast(detection_classes_ptr[i]); - } - MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores, - detection_classes.data(), - output_detections)); - } - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::ProcessGPU( - CalculatorContext* cc, std::vector* output_detections) { - const auto& input_tensors = *kInTensors(cc); - RET_CHECK_GE(input_tensors.size(), 2); -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc, - &output_detections]() - -> absl::Status { - if (!anchors_init_) { - if (input_tensors.size() == kNumInputTensorsWithAnchors) { - auto read_view = input_tensors[2].GetOpenGlBufferReadView(); - glBindBuffer(GL_COPY_READ_BUFFER, read_view.name()); - auto write_view = raw_anchors_buffer_->GetOpenGlBufferWriteView(); - glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name()); - glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, - input_tensors[2].bytes()); - } else if (!kInAnchors(cc).IsEmpty()) { - const auto& anchors = *kInAnchors(cc); - auto anchors_view = raw_anchors_buffer_->GetCpuWriteView(); - auto raw_anchors = anchors_view.buffer(); - ConvertAnchorsToRawValues(anchors, num_boxes_, raw_anchors); - } else { - return absl::UnavailableError("No anchor data available."); - } - anchors_init_ = true; - } - // Use the scope to release the writable buffers' views before requesting - // the reading buffers' views. - { - // Decode boxes. - auto scored_boxes_view = scored_boxes_buffer_->GetOpenGlBufferWriteView(); - auto decoded_boxes_view = - decoded_boxes_buffer_->GetOpenGlBufferWriteView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, decoded_boxes_view.name()); - auto input0_view = input_tensors[0].GetOpenGlBufferReadView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, input0_view.name()); - auto raw_anchors_view = raw_anchors_buffer_->GetOpenGlBufferReadView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, raw_anchors_view.name()); - glUseProgram(decode_program_); - glDispatchCompute(num_boxes_, 1, 1); - - // Score boxes. - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, scored_boxes_view.name()); - auto input1_view = input_tensors[1].GetOpenGlBufferReadView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, input1_view.name()); - glUseProgram(score_program_); - glDispatchCompute(num_boxes_, 1, 1); - } - return absl::OkStatus(); - })); - - // TODO: b/138851969. Is it possible to output a float vector - // for score and an int vector for class so that we can avoid copying twice? - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - // The order of requesting of CpuViews must be the same as the order of - // requesting OpenGlViews above to avoid 'Potential mutex deadlock' message - // when compiled without '-c opt' option. - auto scored_boxes_view = scored_boxes_buffer_->GetCpuReadView(); - auto score_class_id_pairs = scored_boxes_view.buffer(); - for (int i = 0; i < num_boxes_; ++i) { - detection_scores[i] = score_class_id_pairs[i * 2]; - detection_classes[i] = static_cast(score_class_id_pairs[i * 2 + 1]); - } - auto decoded_boxes_view = decoded_boxes_buffer_->GetCpuReadView(); - auto boxes = decoded_boxes_view.buffer(); - MP_RETURN_IF_ERROR(ConvertToDetections(boxes, detection_scores.data(), - detection_classes.data(), - output_detections)); -#elif MEDIAPIPE_METAL_ENABLED - id device = gpu_helper_.mtlDevice; - if (!anchors_init_) { - if (input_tensors.size() == kNumInputTensorsWithAnchors) { - RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); - auto command_buffer = [gpu_helper_ commandBuffer]; - auto src_buffer = input_tensors[2].GetMtlBufferReadView(command_buffer); - auto dest_buffer = - raw_anchors_buffer_->GetMtlBufferWriteView(command_buffer); - id blit_command = - [command_buffer blitCommandEncoder]; - [blit_command copyFromBuffer:src_buffer.buffer() - sourceOffset:0 - toBuffer:dest_buffer.buffer() - destinationOffset:0 - size:input_tensors[2].bytes()]; - [blit_command endEncoding]; - [command_buffer commit]; - } else if (!kInAnchors(cc).IsEmpty()) { - const auto& anchors = *kInAnchors(cc); - auto raw_anchors_view = raw_anchors_buffer_->GetCpuWriteView(); - ConvertAnchorsToRawValues(anchors, num_boxes_, - raw_anchors_view.buffer()); - } else { - return absl::UnavailableError("No anchor data available."); - } - anchors_init_ = true; - } - - // Use the scope to release the writable buffers' views before requesting the - // reading buffers' views. - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"DecodeAndScoreBoxes"; - id command_encoder = - [command_buffer computeCommandEncoder]; - [command_encoder setComputePipelineState:decode_program_]; - { - auto scored_boxes_view = - scored_boxes_buffer_->GetMtlBufferWriteView(command_buffer); - auto decoded_boxes_view = - decoded_boxes_buffer_->GetMtlBufferWriteView(command_buffer); - [command_encoder setBuffer:decoded_boxes_view.buffer() offset:0 atIndex:0]; - auto input0_view = input_tensors[0].GetMtlBufferReadView(command_buffer); - [command_encoder setBuffer:input0_view.buffer() offset:0 atIndex:1]; - auto raw_anchors_view = - raw_anchors_buffer_->GetMtlBufferReadView(command_buffer); - [command_encoder setBuffer:raw_anchors_view.buffer() offset:0 atIndex:2]; - MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1); - MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1); - [command_encoder dispatchThreadgroups:decode_threadgroups - threadsPerThreadgroup:decode_threads_per_group]; - - [command_encoder setComputePipelineState:score_program_]; - [command_encoder setBuffer:scored_boxes_view.buffer() offset:0 atIndex:0]; - auto input1_view = input_tensors[1].GetMtlBufferReadView(command_buffer); - [command_encoder setBuffer:input1_view.buffer() offset:0 atIndex:1]; - MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1); - MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1); - [command_encoder dispatchThreadgroups:score_threadgroups - threadsPerThreadgroup:score_threads_per_group]; - [command_encoder endEncoding]; - [command_buffer commit]; - } - - // Output detections. - // TODO Adjust shader to avoid copying shader output twice. - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - { - auto scored_boxes_view = scored_boxes_buffer_->GetCpuReadView(); - auto score_class_id_pairs = scored_boxes_view.buffer(); - for (int i = 0; i < num_boxes_; ++i) { - detection_scores[i] = score_class_id_pairs[i * 2]; - detection_classes[i] = static_cast(score_class_id_pairs[i * 2 + 1]); - } - } - auto decoded_boxes_view = decoded_boxes_buffer_->GetCpuReadView(); - auto boxes = decoded_boxes_view.buffer(); - MP_RETURN_IF_ERROR(ConvertToDetections(boxes, detection_scores.data(), - detection_classes.data(), - output_detections)); - -#else - LOG(ERROR) << "GPU input on non-Android not supported yet."; -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::Close(CalculatorContext* cc) { -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - gpu_helper_.RunInGlContext([this] { - decoded_boxes_buffer_ = nullptr; - scored_boxes_buffer_ = nullptr; - raw_anchors_buffer_ = nullptr; - glDeleteProgram(decode_program_); - glDeleteProgram(score_program_); - }); -#elif MEDIAPIPE_METAL_ENABLED - decoded_boxes_buffer_ = nullptr; - scored_boxes_buffer_ = nullptr; - raw_anchors_buffer_ = nullptr; - decode_program_ = nil; - score_program_ = nil; -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::LoadOptions(CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToDetectionsCalculatorOptions>(); - RET_CHECK(options_.has_num_classes()); - RET_CHECK(options_.has_num_boxes()); - RET_CHECK(options_.has_num_coords()); - - num_classes_ = options_.num_classes(); - num_boxes_ = options_.num_boxes(); - num_coords_ = options_.num_coords(); - - // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); - - // Check if the output size is equal to the requested boxes and keypoints. - CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + - kNumCoordsPerBox, - num_coords_); - - if (kSideInIgnoreClasses(cc).IsConnected()) { - RET_CHECK(!kSideInIgnoreClasses(cc).IsEmpty()); - for (int ignore_class : *kSideInIgnoreClasses(cc)) { - ignore_classes_.insert(ignore_class); - } - } else { - for (int i = 0; i < options_.ignore_classes_size(); ++i) { - ignore_classes_.insert(options_.ignore_classes(i)); - } - } - - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::DecodeBoxes( - const float* raw_boxes, const std::vector& anchors, - std::vector* boxes) { - for (int i = 0; i < num_boxes_; ++i) { - const int box_offset = i * num_coords_ + options_.box_coord_offset(); - - float y_center = raw_boxes[box_offset]; - float x_center = raw_boxes[box_offset + 1]; - float h = raw_boxes[box_offset + 2]; - float w = raw_boxes[box_offset + 3]; - if (options_.reverse_output_order()) { - x_center = raw_boxes[box_offset]; - y_center = raw_boxes[box_offset + 1]; - w = raw_boxes[box_offset + 2]; - h = raw_boxes[box_offset + 3]; - } - - x_center = - x_center / options_.x_scale() * anchors[i].w() + anchors[i].x_center(); - y_center = - y_center / options_.y_scale() * anchors[i].h() + anchors[i].y_center(); - - if (options_.apply_exponential_on_box_size()) { - h = std::exp(h / options_.h_scale()) * anchors[i].h(); - w = std::exp(w / options_.w_scale()) * anchors[i].w(); - } else { - h = h / options_.h_scale() * anchors[i].h(); - w = w / options_.w_scale() * anchors[i].w(); - } - - const float ymin = y_center - h / 2.f; - const float xmin = x_center - w / 2.f; - const float ymax = y_center + h / 2.f; - const float xmax = x_center + w / 2.f; - - (*boxes)[i * num_coords_ + 0] = ymin; - (*boxes)[i * num_coords_ + 1] = xmin; - (*boxes)[i * num_coords_ + 2] = ymax; - (*boxes)[i * num_coords_ + 3] = xmax; - - if (options_.num_keypoints()) { - for (int k = 0; k < options_.num_keypoints(); ++k) { - const int offset = i * num_coords_ + options_.keypoint_coord_offset() + - k * options_.num_values_per_keypoint(); - - float keypoint_y = raw_boxes[offset]; - float keypoint_x = raw_boxes[offset + 1]; - if (options_.reverse_output_order()) { - keypoint_x = raw_boxes[offset]; - keypoint_y = raw_boxes[offset + 1]; - } - - (*boxes)[offset] = keypoint_x / options_.x_scale() * anchors[i].w() + - anchors[i].x_center(); - (*boxes)[offset + 1] = - keypoint_y / options_.y_scale() * anchors[i].h() + - anchors[i].y_center(); - } - } - } - - return absl::OkStatus(); -} - -absl::Status TensorsToDetectionsCalculator::ConvertToDetections( - const float* detection_boxes, const float* detection_scores, - const int* detection_classes, std::vector* output_detections) { - for (int i = 0; i < num_boxes_; ++i) { - if (options_.has_min_score_thresh() && - detection_scores[i] < options_.min_score_thresh()) { - continue; - } - const int box_offset = i * num_coords_; - Detection detection = ConvertToDetection( - detection_boxes[box_offset + 0], detection_boxes[box_offset + 1], - detection_boxes[box_offset + 2], detection_boxes[box_offset + 3], - detection_scores[i], detection_classes[i], options_.flip_vertically()); - const auto& bbox = detection.location_data().relative_bounding_box(); - if (bbox.width() < 0 || bbox.height() < 0) { - // Decoded detection boxes could have negative values for width/height due - // to model prediction. Filter out those boxes since some downstream - // calculators may assume non-negative values. (b/171391719) - continue; - } - // Add keypoints. - if (options_.num_keypoints() > 0) { - auto* location_data = detection.mutable_location_data(); - for (int kp_id = 0; kp_id < options_.num_keypoints() * - options_.num_values_per_keypoint(); - kp_id += options_.num_values_per_keypoint()) { - auto keypoint = location_data->add_relative_keypoints(); - const int keypoint_index = - box_offset + options_.keypoint_coord_offset() + kp_id; - keypoint->set_x(detection_boxes[keypoint_index + 0]); - keypoint->set_y(options_.flip_vertically() - ? 1.f - detection_boxes[keypoint_index + 1] - : detection_boxes[keypoint_index + 1]); - } - } - output_detections->emplace_back(detection); - } - return absl::OkStatus(); -} - -Detection TensorsToDetectionsCalculator::ConvertToDetection( - float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, - int class_id, bool flip_vertically) { - Detection detection; - detection.add_score(score); - detection.add_label_id(class_id); - - LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - - LocationData::RelativeBoundingBox* relative_bbox = - location_data->mutable_relative_bounding_box(); - - relative_bbox->set_xmin(box_xmin); - relative_bbox->set_ymin(flip_vertically ? 1.f - box_ymax : box_ymin); - relative_bbox->set_width(box_xmax - box_xmin); - relative_bbox->set_height(box_ymax - box_ymin); - return detection; -} - -absl::Status TensorsToDetectionsCalculator::GpuInit(CalculatorContext* cc) { -#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> absl::Status { - // A shader to decode detection boxes. - const std::string decode_src = absl::Substitute( - R"( #version 310 es - -layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in; - -layout(location = 0) uniform vec4 scale; - -layout(std430, binding = 0) writeonly buffer Output { - float data[]; -} boxes; - -layout(std430, binding = 1) readonly buffer Input0 { - float data[]; -} raw_boxes; - -layout(std430, binding = 2) readonly buffer Input1 { - float data[]; -} raw_anchors; - -uint num_coords = uint($0); -int reverse_output_order = int($1); -int apply_exponential = int($2); -int box_coord_offset = int($3); -int num_keypoints = int($4); -int keypt_coord_offset = int($5); -int num_values_per_keypt = int($6); - -void main() { - uint g_idx = gl_GlobalInvocationID.x; // box index - uint box_offset = g_idx * num_coords + uint(box_coord_offset); - uint anchor_offset = g_idx * uint(4); // check kNumCoordsPerBox - - float y_center, x_center, h, w; - - if (reverse_output_order == int(0)) { - y_center = raw_boxes.data[box_offset + uint(0)]; - x_center = raw_boxes.data[box_offset + uint(1)]; - h = raw_boxes.data[box_offset + uint(2)]; - w = raw_boxes.data[box_offset + uint(3)]; - } else { - x_center = raw_boxes.data[box_offset + uint(0)]; - y_center = raw_boxes.data[box_offset + uint(1)]; - w = raw_boxes.data[box_offset + uint(2)]; - h = raw_boxes.data[box_offset + uint(3)]; - } - - float anchor_yc = raw_anchors.data[anchor_offset + uint(0)]; - float anchor_xc = raw_anchors.data[anchor_offset + uint(1)]; - float anchor_h = raw_anchors.data[anchor_offset + uint(2)]; - float anchor_w = raw_anchors.data[anchor_offset + uint(3)]; - - x_center = x_center / scale.x * anchor_w + anchor_xc; - y_center = y_center / scale.y * anchor_h + anchor_yc; - - if (apply_exponential == int(1)) { - h = exp(h / scale.w) * anchor_h; - w = exp(w / scale.z) * anchor_w; - } else { - h = (h / scale.w) * anchor_h; - w = (w / scale.z) * anchor_w; - } - - float ymin = y_center - h / 2.0; - float xmin = x_center - w / 2.0; - float ymax = y_center + h / 2.0; - float xmax = x_center + w / 2.0; - - boxes.data[box_offset + uint(0)] = ymin; - boxes.data[box_offset + uint(1)] = xmin; - boxes.data[box_offset + uint(2)] = ymax; - boxes.data[box_offset + uint(3)] = xmax; - - if (num_keypoints > int(0)){ - for (int k = 0; k < num_keypoints; ++k) { - int kp_offset = - int(g_idx * num_coords) + keypt_coord_offset + k * num_values_per_keypt; - float kp_y, kp_x; - if (reverse_output_order == int(0)) { - kp_y = raw_boxes.data[kp_offset + int(0)]; - kp_x = raw_boxes.data[kp_offset + int(1)]; - } else { - kp_x = raw_boxes.data[kp_offset + int(0)]; - kp_y = raw_boxes.data[kp_offset + int(1)]; - } - boxes.data[kp_offset + int(0)] = kp_x / scale.x * anchor_w + anchor_xc; - boxes.data[kp_offset + int(1)] = kp_y / scale.y * anchor_h + anchor_yc; - } - } -})", - options_.num_coords(), // box xywh - options_.reverse_output_order() ? 1 : 0, - options_.apply_exponential_on_box_size() ? 1 : 0, - options_.box_coord_offset(), options_.num_keypoints(), - options_.keypoint_coord_offset(), options_.num_values_per_keypoint()); - - // Shader program - GLuint shader = glCreateShader(GL_COMPUTE_SHADER); - const GLchar* sources[] = {decode_src.c_str()}; - glShaderSource(shader, 1, sources, NULL); - glCompileShader(shader); - GLint compiled = GL_FALSE; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); - RET_CHECK(compiled == GL_TRUE) << "Shader compilation error: " << [shader] { - GLint length; - glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &length); - std::string str; - str.reserve(length); - glGetShaderInfoLog(shader, length, nullptr, str.data()); - return str; - }(); - decode_program_ = glCreateProgram(); - glAttachShader(decode_program_, shader); - glDeleteShader(shader); - glLinkProgram(decode_program_); - - // Outputs - decoded_boxes_buffer_ = - absl::make_unique(Tensor::ElementType::kFloat32, - Tensor::Shape{1, num_boxes_ * num_coords_}); - raw_anchors_buffer_ = absl::make_unique( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, num_boxes_ * kNumCoordsPerBox}); - // Parameters - glUseProgram(decode_program_); - glUniform4f(0, options_.x_scale(), options_.y_scale(), options_.w_scale(), - options_.h_scale()); - - // A shader to score detection boxes. - const std::string score_src = absl::Substitute( - R"( #version 310 es - -layout(local_size_x = 1, local_size_y = $0, local_size_z = 1) in; - -#define FLT_MAX 1.0e+37 - -shared float local_scores[$0]; - -layout(std430, binding = 0) writeonly buffer Output { - float data[]; -} scored_boxes; - -layout(std430, binding = 1) readonly buffer Input0 { - float data[]; -} raw_scores; - -uint num_classes = uint($0); -int apply_sigmoid = int($1); -int apply_clipping_thresh = int($2); -float clipping_thresh = float($3); -int ignore_class_0 = int($4); - -float optional_sigmoid(float x) { - if (apply_sigmoid == int(0)) return x; - if (apply_clipping_thresh == int(1)) { - x = clamp(x, -clipping_thresh, clipping_thresh); - } - x = 1.0 / (1.0 + exp(-x)); - return x; -} - -void main() { - uint g_idx = gl_GlobalInvocationID.x; // box idx - uint s_idx = gl_LocalInvocationID.y; // score/class idx - - // load all scores into shared memory - float score = raw_scores.data[g_idx * num_classes + s_idx]; - local_scores[s_idx] = optional_sigmoid(score); - memoryBarrierShared(); - barrier(); - - // find max score in shared memory - if (s_idx == uint(0)) { - float max_score = -FLT_MAX; - float max_class = -1.0; - for (int i=ignore_class_0; i max_score) { - max_score = local_scores[i]; - max_class = float(i); - } - } - scored_boxes.data[g_idx * uint(2) + uint(0)] = max_score; - scored_boxes.data[g_idx * uint(2) + uint(1)] = max_class; - } -})", - num_classes_, options_.sigmoid_score() ? 1 : 0, - options_.has_score_clipping_thresh() ? 1 : 0, - options_.has_score_clipping_thresh() ? options_.score_clipping_thresh() - : 0, - !ignore_classes_.empty() ? 1 : 0); - - // # filter classes supported is hardware dependent. - int max_wg_size; // typically <= 1024 - glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1, - &max_wg_size); // y-dim - CHECK_LT(num_classes_, max_wg_size) - << "# classes must be < " << max_wg_size; - // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; - - // Shader program - { - GLuint shader = glCreateShader(GL_COMPUTE_SHADER); - const GLchar* sources[] = {score_src.c_str()}; - glShaderSource(shader, 1, sources, NULL); - glCompileShader(shader); - GLint compiled = GL_FALSE; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); - RET_CHECK(compiled == GL_TRUE); - score_program_ = glCreateProgram(); - glAttachShader(score_program_, shader); - glDeleteShader(shader); - glLinkProgram(score_program_); - } - - // Outputs - scored_boxes_buffer_ = absl::make_unique( - Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2}); - - return absl::OkStatus(); - })); - -#elif MEDIAPIPE_METAL_ENABLED - id device = gpu_helper_.mtlDevice; - - // A shader to decode detection boxes. - std::string decode_src = absl::Substitute( - R"( -#include - -using namespace metal; - -kernel void decodeKernel( - device float* boxes [[ buffer(0) ]], - device float* raw_boxes [[ buffer(1) ]], - device float* raw_anchors [[ buffer(2) ]], - uint2 gid [[ thread_position_in_grid ]]) { - - uint num_coords = uint($0); - int reverse_output_order = int($1); - int apply_exponential = int($2); - int box_coord_offset = int($3); - int num_keypoints = int($4); - int keypt_coord_offset = int($5); - int num_values_per_keypt = int($6); -)", - options_.num_coords(), // box xywh - options_.reverse_output_order() ? 1 : 0, - options_.apply_exponential_on_box_size() ? 1 : 0, - options_.box_coord_offset(), options_.num_keypoints(), - options_.keypoint_coord_offset(), options_.num_values_per_keypoint()); - decode_src += absl::Substitute( - R"( - float4 scale = float4(($0),($1),($2),($3)); -)", - options_.x_scale(), options_.y_scale(), options_.w_scale(), - options_.h_scale()); - decode_src += R"( - uint g_idx = gid.x; - uint box_offset = g_idx * num_coords + uint(box_coord_offset); - uint anchor_offset = g_idx * uint(4); // check kNumCoordsPerBox - - float y_center, x_center, h, w; - - if (reverse_output_order == int(0)) { - y_center = raw_boxes[box_offset + uint(0)]; - x_center = raw_boxes[box_offset + uint(1)]; - h = raw_boxes[box_offset + uint(2)]; - w = raw_boxes[box_offset + uint(3)]; - } else { - x_center = raw_boxes[box_offset + uint(0)]; - y_center = raw_boxes[box_offset + uint(1)]; - w = raw_boxes[box_offset + uint(2)]; - h = raw_boxes[box_offset + uint(3)]; - } - - float anchor_yc = raw_anchors[anchor_offset + uint(0)]; - float anchor_xc = raw_anchors[anchor_offset + uint(1)]; - float anchor_h = raw_anchors[anchor_offset + uint(2)]; - float anchor_w = raw_anchors[anchor_offset + uint(3)]; - - x_center = x_center / scale.x * anchor_w + anchor_xc; - y_center = y_center / scale.y * anchor_h + anchor_yc; - - if (apply_exponential == int(1)) { - h = exp(h / scale.w) * anchor_h; - w = exp(w / scale.z) * anchor_w; - } else { - h = (h / scale.w) * anchor_h; - w = (w / scale.z) * anchor_w; - } - - float ymin = y_center - h / 2.0; - float xmin = x_center - w / 2.0; - float ymax = y_center + h / 2.0; - float xmax = x_center + w / 2.0; - - boxes[box_offset + uint(0)] = ymin; - boxes[box_offset + uint(1)] = xmin; - boxes[box_offset + uint(2)] = ymax; - boxes[box_offset + uint(3)] = xmax; - - if (num_keypoints > int(0)){ - for (int k = 0; k < num_keypoints; ++k) { - int kp_offset = - int(g_idx * num_coords) + keypt_coord_offset + k * num_values_per_keypt; - float kp_y, kp_x; - if (reverse_output_order == int(0)) { - kp_y = raw_boxes[kp_offset + int(0)]; - kp_x = raw_boxes[kp_offset + int(1)]; - } else { - kp_x = raw_boxes[kp_offset + int(0)]; - kp_y = raw_boxes[kp_offset + int(1)]; - } - boxes[kp_offset + int(0)] = kp_x / scale.x * anchor_w + anchor_xc; - boxes[kp_offset + int(1)] = kp_y / scale.y * anchor_h + anchor_yc; - } - } -})"; - - { - // Shader program - NSString* library_source = - [NSString stringWithUTF8String:decode_src.c_str()]; - NSError* error = nil; - id library = [device newLibraryWithSource:library_source - options:nullptr - error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"decodeKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - decode_program_ = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(decode_program_ != nil) << "Couldn't create pipeline state " << - [[error localizedDescription] UTF8String]; - // Outputs - decoded_boxes_buffer_ = - absl::make_unique(Tensor::ElementType::kFloat32, - Tensor::Shape{1, num_boxes_ * num_coords_}); - // Inputs - raw_anchors_buffer_ = absl::make_unique( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, num_boxes_ * kNumCoordsPerBox}); - } - - // A shader to score detection boxes. - const std::string score_src = absl::Substitute( - R"( -#include - -using namespace metal; - -float optional_sigmoid(float x) { - int apply_sigmoid = int($1); - int apply_clipping_thresh = int($2); - float clipping_thresh = float($3); - if (apply_sigmoid == int(0)) return x; - if (apply_clipping_thresh == int(1)) { - x = clamp(x, -clipping_thresh, clipping_thresh); - } - x = 1.0 / (1.0 + exp(-x)); - return x; -} - -kernel void scoreKernel( - device float* scored_boxes [[ buffer(0) ]], - device float* raw_scores [[ buffer(1) ]], - uint2 tid [[ thread_position_in_threadgroup ]], - uint2 gid [[ thread_position_in_grid ]]) { - - uint num_classes = uint($0); - int apply_sigmoid = int($1); - int apply_clipping_thresh = int($2); - float clipping_thresh = float($3); - int ignore_class_0 = int($4); - - uint g_idx = gid.x; // box idx - uint s_idx = tid.y; // score/class idx - - // load all scores into shared memory - threadgroup float local_scores[$0]; - float score = raw_scores[g_idx * num_classes + s_idx]; - local_scores[s_idx] = optional_sigmoid(score); - threadgroup_barrier(mem_flags::mem_threadgroup); - - // find max score in shared memory - if (s_idx == uint(0)) { - float max_score = -FLT_MAX; - float max_class = -1.0; - for (int i=ignore_class_0; i max_score) { - max_score = local_scores[i]; - max_class = float(i); - } - } - scored_boxes[g_idx * uint(2) + uint(0)] = max_score; - scored_boxes[g_idx * uint(2) + uint(1)] = max_class; - } -})", - num_classes_, options_.sigmoid_score() ? 1 : 0, - options_.has_score_clipping_thresh() ? 1 : 0, - options_.has_score_clipping_thresh() ? options_.score_clipping_thresh() - : 0, - ignore_classes_.size() ? 1 : 0); - - // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; - - { - // Shader program - NSString* library_source = - [NSString stringWithUTF8String:score_src.c_str()]; - NSError* error = nil; - id library = [device newLibraryWithSource:library_source - options:nullptr - error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"scoreKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - score_program_ = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(score_program_ != nil) << "Couldn't create pipeline state " << - [[error localizedDescription] UTF8String]; - // Outputs - scored_boxes_buffer_ = absl::make_unique( - Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2}); - // # filter classes supported is hardware dependent. - int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup; - CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size; - } - -#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto b/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto deleted file mode 100644 index 364eb5cce..000000000 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TensorsToDetectionsCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorsToDetectionsCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TensorsToDetectionsCalculatorOptions ext = 335742639; - } - - // [Required] The number of output classes predicted by the detection model. - optional int32 num_classes = 1; - // [Required] The number of output boxes predicted by the detection model. - optional int32 num_boxes = 2; - // [Required] The number of output values per boxes predicted by the detection - // model. The values contain bounding boxes, keypoints, etc. - optional int32 num_coords = 3; - - // The offset of keypoint coordinates in the location tensor. - optional int32 keypoint_coord_offset = 9; - // The number of predicted keypoints. - optional int32 num_keypoints = 10 [default = 0]; - // The dimension of each keypoint, e.g. number of values predicted for each - // keypoint. - optional int32 num_values_per_keypoint = 11 [default = 2]; - // The offset of box coordinates in the location tensor. - optional int32 box_coord_offset = 12 [default = 0]; - - // Parameters for decoding SSD detection model. - optional float x_scale = 4 [default = 0.0]; - optional float y_scale = 5 [default = 0.0]; - optional float w_scale = 6 [default = 0.0]; - optional float h_scale = 7 [default = 0.0]; - - optional bool apply_exponential_on_box_size = 13 [default = false]; - - // Whether to reverse the order of predicted x, y from output. - // If false, the order is [y_center, x_center, h, w], if true the order is - // [x_center, y_center, w, h]. - optional bool reverse_output_order = 14 [default = false]; - // The ids of classes that should be ignored during decoding the score for - // each predicted box. Can be overridden with IGNORE_CLASSES side packet. - repeated int32 ignore_classes = 8; - - optional bool sigmoid_score = 15 [default = false]; - optional float score_clipping_thresh = 16; - - // Whether the detection coordinates from the input tensors should be flipped - // vertically (along the y-direction). This is useful, for example, when the - // input tensors represent detections defined with a coordinate system where - // the origin is at the top-left corner, whereas the desired detection - // representation has a bottom-left origin (e.g., in OpenGL). - optional bool flip_vertically = 18 [default = false]; - - // Score threshold for perserving decoded detections. - optional float min_score_thresh = 19; -} diff --git a/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc b/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc deleted file mode 100644 index 5ec3b4dea..000000000 --- a/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_floats_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -inline float Sigmoid(float value) { return 1.0f / (1.0f + std::exp(-value)); } - -} // namespace - -// A calculator for converting Tensors to to a float or a float vector. -// -// Input: -// TENSORS - Vector of Tensors of type kFloat32. Only the first -// tensor will be used. -// Output: -// FLOAT(optional) - Converted single float number. -// FLOATS(optional) - Converted float vector. -// -// Notes: To output FLOAT stream, the input tensor must have size 1, e.g. -// only 1 float number in the tensor. -// -// Usage example: -// node { -// calculator: "TensorsToFloatsCalculator" -// input_stream: "TENSORS:tensors" -// output_stream: "FLOATS:floats" -// } -namespace api2 { -class TensorsToFloatsCalculator : public Node { - public: - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr Output::Optional kOutFloat{"FLOAT"}; - static constexpr Output>::Optional kOutFloats{"FLOATS"}; - MEDIAPIPE_NODE_INTERFACE(TensorsToFloatsCalculator, kInTensors, kOutFloat, - kOutFloats); - - static absl::Status UpdateContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) final; - absl::Status Process(CalculatorContext* cc) final; - - private: - ::mediapipe::TensorsToFloatsCalculatorOptions options_; -}; -MEDIAPIPE_REGISTER_NODE(TensorsToFloatsCalculator); - -absl::Status TensorsToFloatsCalculator::UpdateContract(CalculatorContract* cc) { - // Only exactly a single output allowed. - RET_CHECK(kOutFloat(cc).IsConnected() ^ kOutFloats(cc).IsConnected()); - return absl::OkStatus(); -} - -absl::Status TensorsToFloatsCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options<::mediapipe::TensorsToFloatsCalculatorOptions>(); - return absl::OkStatus(); -} - -absl::Status TensorsToFloatsCalculator::Process(CalculatorContext* cc) { - const auto& input_tensors = *kInTensors(cc); - RET_CHECK(!input_tensors.empty()); - // TODO: Add option to specify which tensor to take from. - auto view = input_tensors[0].GetCpuReadView(); - auto raw_floats = view.buffer(); - int num_values = input_tensors[0].shape().num_elements(); - auto output_floats = absl::make_unique>( - raw_floats, raw_floats + num_values); - - switch (options_.activation()) { - case TensorsToFloatsCalculatorOptions::SIGMOID: - std::transform(output_floats->begin(), output_floats->end(), - output_floats->begin(), Sigmoid); - break; - case TensorsToFloatsCalculatorOptions::NONE: - break; - } - - if (kOutFloat(cc).IsConnected()) { - // TODO: Could add an index in the option to specifiy returning - // one value of a float array. - RET_CHECK_EQ(num_values, 1); - kOutFloat(cc).Send(output_floats->at(0)); - } else { - kOutFloats(cc).Send(std::move(output_floats)); - } - return absl::OkStatus(); -} -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_floats_calculator.proto b/mediapipe/calculators/tensor/tensors_to_floats_calculator.proto deleted file mode 100644 index 694050190..000000000 --- a/mediapipe/calculators/tensor/tensors_to_floats_calculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TensorsToFloatsCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorsToFloatsCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TensorsToFloatsCalculatorOptions ext = 343499115; - } - enum Activation { - NONE = 0; - SIGMOID = 1; - } - // Apply activation function to the floats. - optional Activation activation = 1 [default = NONE]; -} diff --git a/mediapipe/calculators/tensor/tensors_to_floats_calculator_test.cc b/mediapipe/calculators/tensor/tensors_to_floats_calculator_test.cc deleted file mode 100644 index 0e92baf51..000000000 --- a/mediapipe/calculators/tensor/tensors_to_floats_calculator_test.cc +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/tensors_to_floats_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -using mediapipe::ParseTextProtoOrDie; -using Node = ::mediapipe::CalculatorGraphConfig::Node; - -const float kErrorMargin = 1e-2f; - -class TensorsToFloatsCalculatorTest : public ::testing::Test { - protected: - void BuildGraph(mediapipe::CalculatorRunner* runner, - const std::vector& values) { - auto tensors = absl::make_unique>(); - tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, 1, static_cast(values.size()), 1}); - auto view = tensors->back().GetCpuWriteView(); - float* tensor_buffer = view.buffer(); - ASSERT_NE(tensor_buffer, nullptr); - for (int i = 0; i < values.size(); ++i) { - tensor_buffer[i] = values[i]; - } - - int64 stream_timestamp = 0; - auto& input_stream_packets = - runner->MutableInputs()->Tag("TENSORS").packets; - - input_stream_packets.push_back( - mediapipe::Adopt(tensors.release()) - .At(mediapipe::Timestamp(stream_timestamp++))); - } -}; - -TEST_F(TensorsToFloatsCalculatorTest, SingleValue) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:tensors" - output_stream: "FLOAT:float" - )pb")); - - const float single_value = 0.5; - BuildGraph(&runner, {single_value}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("FLOAT").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& value = output_packets_[0].Get(); - EXPECT_EQ(single_value, value); -} - -TEST_F(TensorsToFloatsCalculatorTest, SingleValueAsVector) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:tensors" - output_stream: "FLOATS:floats" - )pb")); - - const float single_value = 0.5; - BuildGraph(&runner, {single_value}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("FLOATS").packets; - EXPECT_EQ(1, output_packets_.size()); - - const auto& values = output_packets_[0].Get>(); - EXPECT_EQ(1, values.size()); - EXPECT_EQ(single_value, values[0]); -} - -TEST_F(TensorsToFloatsCalculatorTest, FloatVector) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:tensors" - output_stream: "FLOATS:floats" - )pb")); - - const std::vector input_values = {0.f, 0.5f, 1.0f}; - BuildGraph(&runner, input_values); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("FLOATS").packets; - EXPECT_EQ(1, output_packets_.size()); - - const auto& values = output_packets_[0].Get>(); - EXPECT_EQ(input_values.size(), values.size()); - for (int i = 0; i < values.size(); ++i) { - EXPECT_NEAR(values[i], input_values[i], kErrorMargin); - } -} - -TEST_F(TensorsToFloatsCalculatorTest, FloatVectorWithSigmoid) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TensorsToFloatsCalculator" - input_stream: "TENSORS:tensors" - output_stream: "FLOATS:floats" - options { - [mediapipe.TensorsToFloatsCalculatorOptions.ext] { activation: SIGMOID } - } - )pb")); - - const std::vector input_values = {-1.f, 0.f, 1.0f}; - const std::vector expected_output_with_sigmoid = {0.269f, 0.5f, - 0.731f}; - BuildGraph(&runner, input_values); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("FLOATS").packets; - EXPECT_EQ(1, output_packets_.size()); - - const auto& values = output_packets_[0].Get>(); - EXPECT_EQ(expected_output_with_sigmoid.size(), values.size()); - for (int i = 0; i < values.size(); ++i) { - EXPECT_NEAR(values[i], expected_output_with_sigmoid[i], kErrorMargin); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc deleted file mode 100644 index 8e4066bee..000000000 --- a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_landmarks_calculator.pb.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace api2 { - -namespace { - -inline float Sigmoid(float value) { return 1.0f / (1.0f + std::exp(-value)); } - -float ApplyActivation( - ::mediapipe::TensorsToLandmarksCalculatorOptions::Activation activation, - float value) { - switch (activation) { - case ::mediapipe::TensorsToLandmarksCalculatorOptions::SIGMOID: - return Sigmoid(value); - break; - default: - return value; - } -} - -} // namespace - -// A calculator for converting Tensors from regression models into landmarks. -// Note that if the landmarks in the tensor has more than 5 dimensions, only the -// first 5 dimensions will be converted to [x,y,z, visibility, presence]. The -// latter two fields may also stay unset if such attributes are not supported in -// the model. -// -// Input: -// TENSORS - Vector of Tensors of type kFloat32. Only the first tensor will be -// used. The size of the values must be (num_dimension x num_landmarks). -// -// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or -// not. Overrides corresponding side packet and/or field in the calculator -// options. -// -// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. -// Overrides corresponding side packet and/or field in the calculator options. -// -// Input side packet: -// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or -// not. Overrides the corresponding field in the calculator options. -// -// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. -// Overrides the corresponding field in the calculator options. -// -// Output: -// LANDMARKS(optional) - Result MediaPipe landmarks. -// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks. -// -// Notes: -// To output normalized landmarks, user must provide the original input image -// size to the model using calculator option input_image_width and -// input_image_height. -// Usage example: -// node { -// calculator: "TensorsToLandmarksCalculator" -// input_stream: "TENSORS:landmark_tensors" -// output_stream: "LANDMARKS:landmarks" -// output_stream: "NORM_LANDMARKS:landmarks" -// options: { -// [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { -// num_landmarks: 21 -// -// input_image_width: 256 -// input_image_height: 256 -// } -// } -// } -class TensorsToLandmarksCalculator : public Node { - public: - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr Input::SideFallback::Optional kFlipHorizontally{ - "FLIP_HORIZONTALLY"}; - static constexpr Input::SideFallback::Optional kFlipVertically{ - "FLIP_VERTICALLY"}; - static constexpr Output::Optional kOutLandmarkList{"LANDMARKS"}; - static constexpr Output::Optional - kOutNormalizedLandmarkList{"NORM_LANDMARKS"}; - MEDIAPIPE_NODE_CONTRACT(kInTensors, kFlipHorizontally, kFlipVertically, - kOutLandmarkList, kOutNormalizedLandmarkList); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - int num_landmarks_ = 0; - ::mediapipe::TensorsToLandmarksCalculatorOptions options_; -}; -MEDIAPIPE_REGISTER_NODE(TensorsToLandmarksCalculator); - -absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) { - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - if (kOutNormalizedLandmarkList(cc).IsConnected()) { - RET_CHECK(options_.has_input_image_height() && - options_.has_input_image_width()) - << "Must provide input width/height for getting normalized landmarks."; - } - if (kOutLandmarkList(cc).IsConnected() && - (options_.flip_horizontally() || options_.flip_vertically() || - kFlipHorizontally(cc).IsConnected() || - kFlipVertically(cc).IsConnected())) { - RET_CHECK(options_.has_input_image_height() && - options_.has_input_image_width()) - << "Must provide input width/height for using flipping when outputing " - "landmarks in absolute coordinates."; - } - return absl::OkStatus(); -} - -absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) { - if (kInTensors(cc).IsEmpty()) { - return absl::OkStatus(); - } - bool flip_horizontally = - kFlipHorizontally(cc).GetOr(options_.flip_horizontally()); - bool flip_vertically = kFlipVertically(cc).GetOr(options_.flip_vertically()); - - const auto& input_tensors = *kInTensors(cc); - int num_values = input_tensors[0].shape().num_elements(); - const int num_dimensions = num_values / num_landmarks_; - CHECK_GT(num_dimensions, 0); - - auto view = input_tensors[0].GetCpuReadView(); - auto raw_landmarks = view.buffer(); - - LandmarkList output_landmarks; - - for (int ld = 0; ld < num_landmarks_; ++ld) { - const int offset = ld * num_dimensions; - Landmark* landmark = output_landmarks.add_landmark(); - - if (flip_horizontally) { - landmark->set_x(options_.input_image_width() - raw_landmarks[offset]); - } else { - landmark->set_x(raw_landmarks[offset]); - } - if (num_dimensions > 1) { - if (flip_vertically) { - landmark->set_y(options_.input_image_height() - - raw_landmarks[offset + 1]); - } else { - landmark->set_y(raw_landmarks[offset + 1]); - } - } - if (num_dimensions > 2) { - landmark->set_z(raw_landmarks[offset + 2]); - } - if (num_dimensions > 3) { - landmark->set_visibility(ApplyActivation(options_.visibility_activation(), - raw_landmarks[offset + 3])); - } - if (num_dimensions > 4) { - landmark->set_presence(ApplyActivation(options_.presence_activation(), - raw_landmarks[offset + 4])); - } - } - - // Output normalized landmarks if required. - if (kOutNormalizedLandmarkList(cc).IsConnected()) { - NormalizedLandmarkList output_norm_landmarks; - for (int i = 0; i < output_landmarks.landmark_size(); ++i) { - const Landmark& landmark = output_landmarks.landmark(i); - NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark(); - norm_landmark->set_x(landmark.x() / options_.input_image_width()); - norm_landmark->set_y(landmark.y() / options_.input_image_height()); - // Scale Z coordinate as X + allow additional uniform normalization. - norm_landmark->set_z(landmark.z() / options_.input_image_width() / - options_.normalize_z()); - if (landmark.has_visibility()) { // Set only if supported in the model. - norm_landmark->set_visibility(landmark.visibility()); - } - if (landmark.has_presence()) { // Set only if supported in the model. - norm_landmark->set_presence(landmark.presence()); - } - } - kOutNormalizedLandmarkList(cc).Send(std::move(output_norm_landmarks)); - } - - // Output absolute landmarks. - if (kOutLandmarkList(cc).IsConnected()) { - kOutLandmarkList(cc).Send(std::move(output_landmarks)); - } - - return absl::OkStatus(); -} - -absl::Status TensorsToLandmarksCalculator::LoadOptions(CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToLandmarksCalculatorOptions>(); - RET_CHECK(options_.has_num_landmarks()); - num_landmarks_ = options_.num_landmarks(); - - return absl::OkStatus(); -} -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.proto b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.proto deleted file mode 100644 index 2608a1459..000000000 --- a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.proto +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TensorsToLandmarksCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorsToLandmarksCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TensorsToLandmarksCalculatorOptions ext = 335742640; - } - - enum Activation { - NONE = 0; - SIGMOID = 1; - } - - // [Required] Number of landmarks from the output of the model. - optional int32 num_landmarks = 1; - - // Size of the input image for the model. These options are used only when - // normalized landmarks are needed. Z coordinate is scaled as X assuming - // a weak perspective projection camera model. - optional int32 input_image_width = 2; - optional int32 input_image_height = 3; - - // Whether the detection coordinates from the input tensors should be flipped - // vertically (along the y-direction). This is useful, for example, when the - // input tensors represent detections defined with a coordinate system where - // the origin is at the top-left corner, whereas the desired detection - // representation has a bottom-left origin (e.g., in OpenGL). - optional bool flip_vertically = 4 [default = false]; - - // Whether the detection coordinates from the input tensors should be flipped - // horizontally (along the x-direction). This is useful, for example, when the - // input image is horizontally flipped in ImageTransformationCalculator - // beforehand. - optional bool flip_horizontally = 6 [default = false]; - - // A value that Z coordinates should be divided by. This option is used only - // when normalized landmarks are needed. It is applied in addition to Z - // coordinate being re-scaled as X. - optional float normalize_z = 5 [default = 1.0]; - - // Apply activation function to the tensor representing landmark visibility. - optional Activation visibility_activation = 7 [default = NONE]; - - // Apply activation function to the tensor representing landmark presence. - optional Activation presence_activation = 8 [default = NONE]; -} diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc deleted file mode 100644 index 45e242f3c..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ /dev/null @@ -1,882 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/gpu/gpu_origin.pb.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -#include "tensorflow/lite/delegates/gpu/gl/converters/util.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_program.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_texture.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - -#if MEDIAPIPE_METAL_ENABLED -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#endif // MEDIAPIPE_METAL_ENABLED - -namespace { -constexpr int kWorkgroupSize = 8; // Block size for GPU shader. -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; -constexpr char kMaskTag[] = "MASK"; - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} -} // namespace - -namespace mediapipe { - -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -using ::tflite::gpu::gl::GlProgram; -using ::tflite::gpu::gl::GlShader; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - -// Converts Tensors from a tflite segmentation model to an image mask. -// -// Performs optional upscale to OUTPUT_SIZE dimensions if provided, -// otherwise the mask is the same size as input tensor. -// -// If at least one input tensor is already on GPU, processing happens on GPU and -// the output mask is also stored on GPU. Otherwise, processing and the output -// mask are both on CPU. -// -// On GPU, the mask is an RGBA image, in both the R & A channels, scaled 0-1. -// On CPU, the mask is a ImageFormat::VEC32F1 image, with values scaled 0-1. -// -// -// Inputs: -// One of the following TENSORS tags: -// TENSORS: Vector of Tensor, -// The tensor dimensions are specified in this calculator's options. -// OUTPUT_SIZE(optional): std::pair, -// If provided, the size to upscale mask to. -// -// Output: -// MASK: An Image output mask, RGBA(GPU) / VEC32F1(CPU). -// -// Options: -// See tensors_to_segmentation_calculator.proto -// -// Usage example: -// node { -// calculator: "TensorsToSegmentationCalculator" -// input_stream: "TENSORS:tensors" -// input_stream: "OUTPUT_SIZE:size" -// output_stream: "MASK:hair_mask" -// node_options: { -// [mediapipe.TensorsToSegmentationCalculatorOptions] { -// output_layer_index: 1 -// # gpu_origin: CONVENTIONAL # or TOP_LEFT -// } -// } -// } -// -// Currently only OpenGLES 3.1 and CPU backends supported. -// TODO Refactor and add support for other backends/platforms. -// -class TensorsToSegmentationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc); - absl::Status ProcessCpu(CalculatorContext* cc); - void GlRender(); - - bool DoesGpuTextureStartAtBottom() { - return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; - } - - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); - - ::mediapipe::TensorsToSegmentationCalculatorOptions options_; - -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint upsample_program_; -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - std::unique_ptr mask_program_31_; -#else - GLuint mask_program_20_; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -#if MEDIAPIPE_METAL_ENABLED - MPPMetalHelper* metal_helper_ = nullptr; - id mask_program_; -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(TensorsToSegmentationCalculator); - -// static -absl::Status TensorsToSegmentationCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - // Inputs. - cc->Inputs().Tag(kTensorsTag).Set>(); - if (cc->Inputs().HasTag(kOutputSizeTag)) { - cc->Inputs().Tag(kOutputSizeTag).Set>(); - } - - // Outputs. - cc->Outputs().Tag(kMaskTag).Set(); - - if (CanUseGpu()) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#if MEDIAPIPE_METAL_ENABLED - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status TensorsToSegmentationCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - bool use_gpu = false; - - if (CanUseGpu()) { -#if !MEDIAPIPE_DISABLE_GPU - use_gpu = true; - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#if MEDIAPIPE_METAL_ENABLED - metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(metal_helper_); -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU - } - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(InitGpu(cc)); -#else - RET_CHECK_FAIL() << "GPU processing disabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - - bool use_gpu = false; - if (CanUseGpu()) { - // Use GPU processing only if at least one input tensor is already on GPU. - for (const auto& tensor : input_tensors) { - if (tensor.ready_on_gpu()) { - use_gpu = true; - break; - } - } - } - - // Validate tensor channels and activation type. - { - RET_CHECK(!input_tensors.empty()); - ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - int tensor_channels = std::get<2>(hwc); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - switch (options_.activation()) { - case Options::NONE: - RET_CHECK_EQ(tensor_channels, 1); - break; - case Options::SIGMOID: - RET_CHECK_EQ(tensor_channels, 1); - break; - case Options::SOFTMAX: - RET_CHECK_EQ(tensor_channels, 2); - break; - } - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(ProcessGpu(cc)); - return absl::OkStatus(); - })); -#else - RET_CHECK_FAIL() << "GPU processing disabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(ProcessCpu(cc)); - } - - return absl::OkStatus(); -} - -absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (upsample_program_) glDeleteProgram(upsample_program_); - upsample_program_ = 0; -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - mask_program_31_.reset(); -#else - if (mask_program_20_) glDeleteProgram(mask_program_20_); - mask_program_20_ = 0; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 -#if MEDIAPIPE_METAL_ENABLED - mask_program_ = nil; -#endif // MEDIAPIPE_METAL_ENABLED - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status TensorsToSegmentationCalculator::ProcessCpu( - CalculatorContext* cc) { - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - std::unique_ptr output_mask = absl::make_unique(mask_frame); - cv::Mat output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, output_mat, cv::Size(output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -template -absl::Status TensorsToSegmentationCalculator::ApplyActivation( - cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} - -// Steps: -// 1. receive tensor -// 2. process segmentation tensor into small mask -// 3. upsample small mask into output mask to be same size as input image -absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - - // Create initial working mask texture. -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - tflite::gpu::gl::GlTexture small_mask_texture; -#else - mediapipe::GlTexture small_mask_texture; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - - // Run shader, process mask tensor. -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - { - MP_RETURN_IF_ERROR(CreateReadWriteRgbaImageTexture( - tflite::gpu::DataType::UINT8, // GL_RGBA8 - {tensor_width, tensor_height}, &small_mask_texture)); - - const int output_index = 0; - glBindImageTexture(output_index, small_mask_texture.id(), 0, GL_FALSE, 0, - GL_WRITE_ONLY, GL_RGBA8); - - auto read_view = input_tensors[0].GetOpenGlBufferReadView(); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, read_view.name()); - - const tflite::gpu::uint3 workgroups = { - NumGroups(tensor_width, kWorkgroupSize), - NumGroups(tensor_height, kWorkgroupSize), 1}; - - glUseProgram(mask_program_31_->id()); - glUniform2i(glGetUniformLocation(mask_program_31_->id(), "out_size"), - tensor_width, tensor_height); - - MP_RETURN_IF_ERROR(mask_program_31_->Dispatch(workgroups)); - } -#elif MEDIAPIPE_METAL_ENABLED - { - id command_buffer = [metal_helper_ commandBuffer]; - command_buffer.label = @"SegmentationKernel"; - id command_encoder = - [command_buffer computeCommandEncoder]; - [command_encoder setComputePipelineState:mask_program_]; - - auto read_view = input_tensors[0].GetMtlBufferReadView(command_buffer); - [command_encoder setBuffer:read_view.buffer() offset:0 atIndex:0]; - - mediapipe::GpuBuffer small_mask_buffer = [metal_helper_ - mediapipeGpuBufferWithWidth:tensor_width - height:tensor_height - format:mediapipe::GpuBufferFormat::kBGRA32]; - id small_mask_texture_metal = - [metal_helper_ metalTextureWithGpuBuffer:small_mask_buffer]; - [command_encoder setTexture:small_mask_texture_metal atIndex:1]; - - unsigned int out_size[] = {static_cast(tensor_width), - static_cast(tensor_height)}; - [command_encoder setBytes:&out_size length:sizeof(out_size) atIndex:2]; - - MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1); - MTLSize threadgroups = - MTLSizeMake(NumGroups(tensor_width, kWorkgroupSize), - NumGroups(tensor_height, kWorkgroupSize), 1); - [command_encoder dispatchThreadgroups:threadgroups - threadsPerThreadgroup:threads_per_group]; - [command_encoder endEncoding]; - [command_buffer commit]; - - small_mask_texture = gpu_helper_.CreateSourceTexture(small_mask_buffer); - } -#else - { - small_mask_texture = gpu_helper_.CreateDestinationTexture( - tensor_width, tensor_height, - mediapipe::GpuBufferFormat::kBGRA32); // actually GL_RGBA8 - - // Go through CPU if not already texture 2D (no direct conversion yet). - // Tensor::GetOpenGlTexture2dReadView() doesn't automatically convert types. - if (!input_tensors[0].ready_as_opengl_texture_2d()) { - (void)input_tensors[0].GetCpuReadView(); - } - - auto read_view = input_tensors[0].GetOpenGlTexture2dReadView(); - - gpu_helper_.BindFramebuffer(small_mask_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, read_view.name()); - glUseProgram(mask_program_20_); - GlRender(); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - - // Upsample small mask into output. - mediapipe::GlTexture output_texture = gpu_helper_.CreateDestinationTexture( - output_width, output_height, - mediapipe::GpuBufferFormat::kBGRA32); // actually GL_RGBA8 - - // Run shader, upsample result. - { - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - glBindTexture(GL_TEXTURE_2D, small_mask_texture.id()); -#else - glBindTexture(GL_TEXTURE_2D, small_mask_texture.name()); -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - glUseProgram(upsample_program_); - GlRender(); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } - - // Send out image as GPU packet. - auto output_image = output_texture.GetFrame(); - cc->Outputs().Tag(kMaskTag).Add(output_image.release(), cc->InputTimestamp()); - - // Cleanup - output_texture.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void TensorsToSegmentationCalculator::GlRender() { -#if !MEDIAPIPE_DISABLE_GPU - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status TensorsToSegmentationCalculator::LoadOptions( - CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); - - return absl::OkStatus(); -} - -absl::Status TensorsToSegmentationCalculator::InitGpu(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> absl::Status { - // A shader to process a segmentation tensor into an output mask. - // Currently uses 4 channels for output, and sets R+A channels as mask value. -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - // GLES 3.1 - const tflite::gpu::uint3 workgroup_size = {kWorkgroupSize, kWorkgroupSize, - 1}; - const std::string shader_header = - absl::StrCat(tflite::gpu::gl::GetShaderHeader(workgroup_size), R"( -precision highp float; - -layout(rgba8, binding = 0) writeonly uniform highp image2D output_texture; - -uniform ivec2 out_size; -)"); - /* Shader defines will be inserted here. */ - - const std::string shader_src_main = R"( -layout(std430, binding = 2) readonly buffer B0 { -#ifdef TWO_CHANNEL_INPUT - vec2 elements[]; -#else - float elements[]; -#endif // TWO_CHANNEL_INPUT -} input_data; // data tensor - -void main() { - int out_width = out_size.x; - int out_height = out_size.y; - - ivec2 gid = ivec2(gl_GlobalInvocationID.xy); - if (gid.x >= out_width || gid.y >= out_height) { return; } - int linear_index = gid.y * out_width + gid.x; - -#ifdef TWO_CHANNEL_INPUT - vec2 input_value = input_data.elements[linear_index]; -#else - vec2 input_value = vec2(input_data.elements[linear_index], 0.0); -#endif // TWO_CHANNEL_INPUT - -// Run activation function. -// One and only one of FN_SOFTMAX,FN_SIGMOID,FN_NONE will be defined. -#ifdef FN_SOFTMAX - // Only two channel input tensor is supported. - vec2 input_px = input_value.rg; - float shift = max(input_px.r, input_px.g); - float softmax_denom = exp(input_px.r - shift) + exp(input_px.g - shift); - float new_mask_value = - exp(input_px[OUTPUT_LAYER_INDEX] - shift) / softmax_denom; -#endif // FN_SOFTMAX - -#ifdef FN_SIGMOID - float new_mask_value = 1.0 / (exp(-input_value.r) + 1.0); -#endif // FN_SIGMOID - -#ifdef FN_NONE - float new_mask_value = input_value.r; -#endif // FN_NONE - -#ifdef FLIP_Y_COORD - int y_coord = out_height - gid.y - 1; -#else - int y_coord = gid.y; -#endif // defined(FLIP_Y_COORD) - ivec2 output_coordinate = ivec2(gid.x, y_coord); - - vec4 out_value = vec4(new_mask_value, 0.0, 0.0, new_mask_value); - imageStore(output_texture, output_coordinate, out_value); -})"; - -#elif MEDIAPIPE_METAL_ENABLED - // METAL - const std::string shader_header = R"( -#include -using namespace metal; -)"; - /* Shader defines will be inserted here. */ - - const std::string shader_src_main = R"( -kernel void segmentationKernel( -#ifdef TWO_CHANNEL_INPUT - device float2* elements [[ buffer(0) ]], -#else - device float* elements [[ buffer(0) ]], -#endif // TWO_CHANNEL_INPUT - texture2d output_texture [[ texture(1) ]], - constant uint* out_size [[ buffer(2) ]], - uint2 gid [[ thread_position_in_grid ]]) -{ - uint out_width = out_size[0]; - uint out_height = out_size[1]; - - if (gid.x >= out_width || gid.y >= out_height) { return; } - uint linear_index = gid.y * out_width + gid.x; - -#ifdef TWO_CHANNEL_INPUT - float2 input_value = elements[linear_index]; -#else - float2 input_value = float2(elements[linear_index], 0.0); -#endif // TWO_CHANNEL_INPUT - -// Run activation function. -// One and only one of FN_SOFTMAX,FN_SIGMOID,FN_NONE will be defined. -#ifdef FN_SOFTMAX - // Only two channel input tensor is supported. - float2 input_px = input_value.xy; - float shift = max(input_px.x, input_px.y); - float softmax_denom = exp(input_px.r - shift) + exp(input_px.g - shift); - float new_mask_value = - exp(input_px[OUTPUT_LAYER_INDEX] - shift) / softmax_denom; -#endif // FN_SOFTMAX - -#ifdef FN_SIGMOID - float new_mask_value = 1.0 / (exp(-input_value.x) + 1.0); -#endif // FN_SIGMOID - -#ifdef FN_NONE - float new_mask_value = input_value.x; -#endif // FN_NONE - -#ifdef FLIP_Y_COORD - int y_coord = out_height - gid.y - 1; -#else - int y_coord = gid.y; -#endif // defined(FLIP_Y_COORD) - uint2 output_coordinate = uint2(gid.x, y_coord); - - float4 out_value = float4(new_mask_value, 0.0, 0.0, new_mask_value); - output_texture.write(out_value, output_coordinate); -} -)"; - -#else - // GLES 2.0 - const std::string shader_header = absl::StrCat( - std::string(mediapipe::kMediaPipeFragmentShaderPreamble), R"( -DEFAULT_PRECISION(mediump, float) -)"); - /* Shader defines will be inserted here. */ - - const std::string shader_src_main = R"( -in vec2 sample_coordinate; - -uniform sampler2D input_texture; - -#ifdef GL_ES -#define fragColor gl_FragColor -#else -out vec4 fragColor; -#endif // defined(GL_ES); - -void main() { -#ifdef FLIP_Y_COORD - float y_coord = 1.0 - sample_coordinate.y; -#else - float y_coord = sample_coordinate.y; -#endif // defined(FLIP_Y_COORD) - vec2 adjusted_coordinate = vec2(sample_coordinate.x, y_coord); - vec4 input_value = texture2D(input_texture, adjusted_coordinate); - - // Run activation function. - // One and only one of FN_SOFTMAX,FN_SIGMOID,FN_NONE will be defined. - -#ifdef FN_SOFTMAX - // Only two channel input tensor is supported. - vec2 input_px = input_value.rg; - float shift = max(input_px.r, input_px.g); - float softmax_denom = exp(input_px.r - shift) + exp(input_px.g - shift); - float new_mask_value = - exp(mix(input_px.r, input_px.g, float(OUTPUT_LAYER_INDEX)) - shift) / softmax_denom; -#endif // FN_SOFTMAX - -#ifdef FN_SIGMOID - float new_mask_value = 1.0 / (exp(-input_value.r) + 1.0); -#endif // FN_SIGMOID - -#ifdef FN_NONE - float new_mask_value = input_value.r; -#endif // FN_NONE - - vec4 out_value = vec4(new_mask_value, 0.0, 0.0, new_mask_value); - fragColor = out_value; -})"; -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - - // Shader defines. - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - const std::string output_layer_index = - "\n#define OUTPUT_LAYER_INDEX int(" + - std::to_string(options_.output_layer_index()) + ")"; - const std::string flip_y_coord = - DoesGpuTextureStartAtBottom() ? "\n#define FLIP_Y_COORD" : ""; - const std::string fn_none = - options_.activation() == Options::NONE ? "\n#define FN_NONE" : ""; - const std::string fn_sigmoid = - options_.activation() == Options::SIGMOID ? "\n#define FN_SIGMOID" : ""; - const std::string fn_softmax = - options_.activation() == Options::SOFTMAX ? "\n#define FN_SOFTMAX" : ""; - const std::string two_channel = options_.activation() == Options::SOFTMAX - ? "\n#define TWO_CHANNEL_INPUT" - : ""; - const std::string shader_defines = - absl::StrCat(output_layer_index, flip_y_coord, fn_softmax, fn_sigmoid, - fn_none, two_channel); - - // Build full shader. - const std::string shader_src_no_previous = - absl::StrCat(shader_header, shader_defines, shader_src_main); - - // Vertex shader attributes. - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Main shader program & parameters -#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - GlShader shader_without_previous; - MP_RETURN_IF_ERROR(GlShader::CompileShader( - GL_COMPUTE_SHADER, shader_src_no_previous, &shader_without_previous)); - mask_program_31_ = absl::make_unique(); - MP_RETURN_IF_ERROR(GlProgram::CreateWithShader(shader_without_previous, - mask_program_31_.get())); -#elif MEDIAPIPE_METAL_ENABLED - id device = metal_helper_.mtlDevice; - NSString* library_source = - [NSString stringWithUTF8String:shader_src_no_previous.c_str()]; - NSError* error = nil; - id library = [device newLibraryWithSource:library_source - options:nullptr - error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"segmentationKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - mask_program_ = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(mask_program_ != nil) << "Couldn't create pipeline state " << - [[error localizedDescription] UTF8String]; -#else - mediapipe::GlhCreateProgram( - mediapipe::kBasicVertexShader, shader_src_no_previous.c_str(), - NUM_ATTRIBUTES, &attr_name[0], attr_location, &mask_program_20_); - RET_CHECK(mask_program_20_) << "Problem initializing the program."; - glUseProgram(mask_program_20_); - glUniform1i(glGetUniformLocation(mask_program_20_, "input_texture"), 1); -#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - - // Simple pass-through program, used for hardware upsampling. - mediapipe::GlhCreateProgram( - mediapipe::kBasicVertexShader, mediapipe::kBasicTexturedFragmentShader, - NUM_ATTRIBUTES, &attr_name[0], attr_location, &upsample_program_); - RET_CHECK(upsample_program_) << "Problem initializing the program."; - glUseProgram(upsample_program_); - glUniform1i(glGetUniformLocation(upsample_program_, "video_frame"), 1); - - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.proto b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.proto deleted file mode 100644 index 1662576b6..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/gpu/gpu_origin.proto"; - -message TensorsToSegmentationCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorsToSegmentationCalculatorOptions ext = 374311106; - } - - // For CONVENTIONAL mode in OpenGL, textures start at bottom and needs - // to be flipped vertically as tensors are expected to start at top. - // (DEFAULT or unset is interpreted as CONVENTIONAL.) - optional GpuOrigin.Mode gpu_origin = 1; - - // Supported activation functions for filtering. - enum Activation { - NONE = 0; // Assumes 1-channel input tensor. - SIGMOID = 1; // Assumes 1-channel input tensor. - SOFTMAX = 2; // Assumes 2-channel input tensor. - } - // Activation function to apply to input tensor. - // Softmax requires a 2-channel tensor, see output_layer_index below. - optional Activation activation = 2 [default = NONE]; - - // Channel to use for processing tensor. - // Only applies when using activation=SOFTMAX. - // Works on two channel input tensor only. - optional int32 output_layer_index = 3 [default = 1]; -} diff --git a/mediapipe/calculators/tensor/testdata/add.bin b/mediapipe/calculators/tensor/testdata/add.bin deleted file mode 100644 index b4c02350c..000000000 Binary files a/mediapipe/calculators/tensor/testdata/add.bin and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/expected_detection.pbtxt b/mediapipe/calculators/tensor/testdata/expected_detection.pbtxt deleted file mode 100644 index f1739ce86..000000000 --- a/mediapipe/calculators/tensor/testdata/expected_detection.pbtxt +++ /dev/null @@ -1,35 +0,0 @@ -label_id: 0 -score: 0.92843366 -location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: 0.21061149 - ymin: 0.29150677 - width: 0.5657704 - height: 0.5657307 - } - relative_keypoints { - x: 0.37730268 - y: 0.44038114 - } - relative_keypoints { - x: 0.6250565 - y: 0.44425336 - } - relative_keypoints { - x: 0.50687385 - y: 0.5767085 - } - relative_keypoints { - x: 0.50173956 - y: 0.6991459 - } - relative_keypoints { - x: 0.2383742 - y: 0.49879026 - } - relative_keypoints { - x: 0.7404449 - y: 0.50361776 - } -} diff --git a/mediapipe/calculators/tensor/testdata/face_detection_expected.png b/mediapipe/calculators/tensor/testdata/face_detection_expected.png deleted file mode 100644 index df38abf70..000000000 Binary files a/mediapipe/calculators/tensor/testdata/face_detection_expected.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt b/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt deleted file mode 100644 index b0e00346c..000000000 --- a/mediapipe/calculators/tensor/testdata/face_detection_test.pbtxt +++ /dev/null @@ -1,31 +0,0 @@ -input_stream: "image" -output_stream: "rendering" -output_stream: "detections" - -# Subgraph that detects faces. -node { - calculator: "FaceDetectionFrontCpu" - input_stream: "IMAGE:image" - output_stream: "DETECTIONS:detections" -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "RENDER_DATA:render_data" - options: { - [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:image" - input_stream: "render_data" - output_stream: "IMAGE:rendering" -} diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/input.jpg b/mediapipe/calculators/tensor/testdata/image_to_tensor/input.jpg deleted file mode 100644 index 37d6c4b20..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/input.jpg and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect.png deleted file mode 100644 index 38a13dabe..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png deleted file mode 100644 index 1a738a50d..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png deleted file mode 100644 index 254dc72ae..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png deleted file mode 100644 index 5b096cb4d..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png deleted file mode 100644 index 104cb6091..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png deleted file mode 100644 index c5512ec0d..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/large_sub_rect_keep_aspect_with_rotation_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png deleted file mode 100644 index aba8d2591..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png deleted file mode 100644 index bfb461546..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png deleted file mode 100644 index 5ce7c3ec3..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png deleted file mode 100644 index ab14e5954..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_keep_aspect_with_rotation_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png deleted file mode 100644 index ecfb1e537..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png deleted file mode 100644 index d55301146..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/image_to_tensor/noop_except_range.png b/mediapipe/calculators/tensor/testdata/image_to_tensor/noop_except_range.png deleted file mode 100644 index 1486d9f15..000000000 Binary files a/mediapipe/calculators/tensor/testdata/image_to_tensor/noop_except_range.png and /dev/null differ diff --git a/mediapipe/calculators/tensor/testdata/labelmap.txt b/mediapipe/calculators/tensor/testdata/labelmap.txt deleted file mode 100644 index 4291e3c6b..000000000 --- a/mediapipe/calculators/tensor/testdata/labelmap.txt +++ /dev/null @@ -1,3 +0,0 @@ -classA -classB -classC diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD deleted file mode 100644 index 0dbbd57da..000000000 --- a/mediapipe/calculators/tensorflow/BUILD +++ /dev/null @@ -1,1192 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -proto_library( - name = "graph_tensors_packet_generator_proto", - srcs = ["graph_tensors_packet_generator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/framework:packet_generator_proto", - ], -) - -proto_library( - name = "matrix_to_tensor_calculator_options_proto", - srcs = ["matrix_to_tensor_calculator_options.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "lapped_tensor_buffer_calculator_proto", - srcs = ["lapped_tensor_buffer_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "object_detection_tensors_to_detections_calculator_proto", - srcs = ["object_detection_tensors_to_detections_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "tensorflow_inference_calculator_proto", - srcs = ["tensorflow_inference_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "tensor_squeeze_dimensions_calculator_proto", - srcs = ["tensor_squeeze_dimensions_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "tensor_to_image_frame_calculator_proto", - srcs = ["tensor_to_image_frame_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "tensor_to_matrix_calculator_proto", - srcs = ["tensor_to_matrix_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/framework/formats:time_series_header_proto", - ], -) - -proto_library( - name = "tensor_to_vector_float_calculator_options_proto", - srcs = ["tensor_to_vector_float_calculator_options.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "unpack_media_sequence_calculator_proto", - srcs = ["unpack_media_sequence_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/calculators/core:packet_resampler_calculator_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:audio_decoder_proto", - ], -) - -proto_library( - name = "vector_float_to_tensor_calculator_options_proto", - srcs = ["vector_float_to_tensor_calculator_options.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "vector_string_to_tensor_calculator_options_proto", - srcs = ["vector_string_to_tensor_calculator_options.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "graph_tensors_packet_generator_cc_proto", - srcs = ["graph_tensors_packet_generator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":graph_tensors_packet_generator_proto"], -) - -mediapipe_cc_proto_library( - name = "image_frame_to_tensor_calculator_cc_proto", - srcs = ["image_frame_to_tensor_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":image_frame_to_tensor_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "matrix_to_tensor_calculator_options_cc_proto", - srcs = ["matrix_to_tensor_calculator_options.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":matrix_to_tensor_calculator_options_proto"], -) - -mediapipe_cc_proto_library( - name = "lapped_tensor_buffer_calculator_cc_proto", - srcs = ["lapped_tensor_buffer_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":lapped_tensor_buffer_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "object_detection_tensors_to_detections_calculator_cc_proto", - srcs = ["object_detection_tensors_to_detections_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":object_detection_tensors_to_detections_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "pack_media_sequence_calculator_cc_proto", - srcs = ["pack_media_sequence_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":pack_media_sequence_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensorflow_inference_calculator_cc_proto", - srcs = ["tensorflow_inference_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tensorflow_inference_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensorflow_session_from_frozen_graph_generator_cc_proto", - srcs = ["tensorflow_session_from_frozen_graph_generator.proto"], - cc_deps = [ - "//mediapipe/framework:packet_generator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":tensorflow_session_from_frozen_graph_generator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensorflow_session_from_frozen_graph_calculator_cc_proto", - srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":tensorflow_session_from_frozen_graph_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensorflow_session_from_saved_model_generator_cc_proto", - srcs = ["tensorflow_session_from_saved_model_generator.proto"], - cc_deps = [ - "//mediapipe/framework:packet_generator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":tensorflow_session_from_saved_model_generator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensorflow_session_from_saved_model_calculator_cc_proto", - srcs = ["tensorflow_session_from_saved_model_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":tensorflow_session_from_saved_model_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensor_squeeze_dimensions_calculator_cc_proto", - srcs = ["tensor_squeeze_dimensions_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tensor_squeeze_dimensions_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensor_to_image_frame_calculator_cc_proto", - srcs = ["tensor_to_image_frame_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tensor_to_image_frame_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensor_to_matrix_calculator_cc_proto", - srcs = ["tensor_to_matrix_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/formats:time_series_header_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":tensor_to_matrix_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tensor_to_vector_float_calculator_options_cc_proto", - srcs = ["tensor_to_vector_float_calculator_options.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tensor_to_vector_float_calculator_options_proto"], -) - -mediapipe_cc_proto_library( - name = "unpack_media_sequence_calculator_cc_proto", - srcs = ["unpack_media_sequence_calculator.proto"], - cc_deps = [ - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:audio_decoder_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":unpack_media_sequence_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "vector_int_to_tensor_calculator_options_cc_proto", - srcs = ["vector_int_to_tensor_calculator_options.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - visibility = ["//visibility:public"], - deps = [":vector_int_to_tensor_calculator_options_proto"], -) - -mediapipe_cc_proto_library( - name = "vector_float_to_tensor_calculator_options_cc_proto", - srcs = ["vector_float_to_tensor_calculator_options.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":vector_float_to_tensor_calculator_options_proto"], -) - -mediapipe_cc_proto_library( - name = "vector_string_to_tensor_calculator_options_cc_proto", - srcs = ["vector_string_to_tensor_calculator_options.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":vector_string_to_tensor_calculator_options_proto"], -) - -cc_library( - name = "graph_tensors_packet_generator", - srcs = ["graph_tensors_packet_generator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":graph_tensors_packet_generator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "image_frame_to_tensor_calculator", - srcs = ["image_frame_to_tensor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":image_frame_to_tensor_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - ], - }), - alwayslink = 1, -) - -cc_library( - name = "matrix_to_tensor_calculator", - srcs = ["matrix_to_tensor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/formats:time_series_header_cc_proto", - ":matrix_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "lapped_tensor_buffer_calculator", - srcs = ["lapped_tensor_buffer_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":lapped_tensor_buffer_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/profiler:circular_buffer", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/types:span", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - ], - alwayslink = 1, -) - -cc_library( - name = "object_detection_tensors_to_detections_calculator", - srcs = ["object_detection_tensors_to_detections_calculator.cc"], - features = [ - # Layering check doesn't play nicely with portable proto wrappers. - "no_layering_check", - ], - visibility = [ - "//visibility:public", - ], - deps = [ - ":object_detection_tensors_to_detections_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:tensor_to_detection", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "pack_media_sequence_calculator", - srcs = ["pack_media_sequence_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util/sequence:media_sequence", - "//mediapipe/util/sequence:media_sequence_util", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - -cc_library( - name = "string_to_sequence_example_calculator", - srcs = ["string_to_sequence_example_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - -# On android, this calculator is configured to run with lite protos. Therefore, -# compile your binary with the flag TENSORFLOW_PROTOS=lite. -cc_library( - name = "tensorflow_inference_calculator", - srcs = ["tensorflow_inference_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tensorflow_session", - ":tensorflow_inference_calculator_cc_proto", - "//mediapipe/framework:timestamp", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework:packet", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tensorflow_session", - hdrs = [ - "tensorflow_session.h", - ], - features = ["no_layering_check"], - visibility = ["//visibility:public"], - deps = select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:core", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - ], - }), -) - -cc_library( - name = "tensorflow_session_from_frozen_graph_calculator", - srcs = ["tensorflow_session_from_frozen_graph_calculator.cc"], - features = ["no_layering_check"], - visibility = ["//visibility:public"], - deps = [ - ":tensorflow_session", - "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - "@org_tensorflow//tensorflow/core:core", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - "//mediapipe/android/file/base", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - "//mediapipe/android/file/base", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tensorflow_session_from_frozen_graph_generator", - srcs = ["tensorflow_session_from_frozen_graph_generator.cc"], - features = ["no_layering_check"], - visibility = ["//visibility:public"], - deps = [ - ":tensorflow_session", - ":tensorflow_session_from_frozen_graph_generator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/port:status", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - "@org_tensorflow//tensorflow/core:core", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - "//mediapipe/android/file/base", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - "//mediapipe/android/file/base", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tensorflow_session_from_saved_model_calculator", - srcs = ["tensorflow_session_from_saved_model_calculator.cc"], - defines = select({ - "//mediapipe:android": ["__ANDROID__"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensorflow_session", - ":tensorflow_session_from_saved_model_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/cc/saved_model:constants", - "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", - "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tensorflow_session_from_saved_model_generator", - srcs = ["tensorflow_session_from_saved_model_generator.cc"], - defines = select({ - "//mediapipe:android": ["__ANDROID__"], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tensorflow_session", - ":tensorflow_session_from_saved_model_generator_cc_proto", - "//mediapipe/framework:packet_generator", - "//mediapipe/framework:packet_type", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/cc/saved_model:constants", - "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", - "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tensor_squeeze_dimensions_calculator", - srcs = ["tensor_squeeze_dimensions_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tensor_squeeze_dimensions_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "tensor_to_image_frame_calculator", - srcs = ["tensor_to_image_frame_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tensor_to_image_frame_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "tensor_to_matrix_calculator", - srcs = ["tensor_to_matrix_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/formats:time_series_header_cc_proto", - ":tensor_to_matrix_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tfrecord_reader_calculator", - srcs = ["tfrecord_reader_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - -cc_library( - name = "tensor_to_vector_float_calculator", - srcs = ["tensor_to_vector_float_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:ret_check", - ":tensor_to_vector_float_calculator_options_cc_proto", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:framework", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "unpack_media_sequence_calculator", - srcs = ["unpack_media_sequence_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:unpack_media_sequence_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:audio_decoder_cc_proto", - "//mediapipe/util/sequence:media_sequence", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - -cc_library( - name = "vector_int_to_tensor_calculator", - srcs = ["vector_int_to_tensor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":vector_int_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "vector_float_to_tensor_calculator", - srcs = ["vector_float_to_tensor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":vector_float_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "vector_string_to_tensor_calculator", - srcs = ["vector_string_to_tensor_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":vector_string_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "unpack_yt8m_sequence_example_calculator", - srcs = ["unpack_yt8m_sequence_example_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":lapped_tensor_buffer_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], - alwayslink = 1, -) - -cc_test( - name = "graph_tensors_packet_generator_test", - srcs = ["graph_tensors_packet_generator_test.cc"], - linkstatic = 1, - deps = [ - ":graph_tensors_packet_generator", - ":graph_tensors_packet_generator_cc_proto", - "//mediapipe/framework:packet", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:packet_set", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/tool:validate_type", - "@org_tensorflow//tensorflow/core:framework", - ], -) - -cc_test( - name = "image_frame_to_tensor_calculator_test", - size = "small", - srcs = ["image_frame_to_tensor_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":image_frame_to_tensor_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:framework", - ], -) - -cc_test( - name = "matrix_to_tensor_calculator_test", - size = "small", - srcs = ["matrix_to_tensor_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":matrix_to_tensor_calculator", - ":matrix_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "@org_tensorflow//tensorflow/core:framework", - ], -) - -cc_test( - name = "lapped_tensor_buffer_calculator_test", - size = "small", - srcs = ["lapped_tensor_buffer_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":lapped_tensor_buffer_calculator", - ":lapped_tensor_buffer_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/memory", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "object_detection_tensors_to_detections_calculator_test", - srcs = ["object_detection_tensors_to_detections_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":object_detection_tensors_to_detections_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:testlib", - ], -) - -cc_test( - name = "pack_media_sequence_calculator_test", - srcs = ["pack_media_sequence_calculator_test.cc"], - deps = [ - ":pack_media_sequence_calculator", - "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/util/sequence:media_sequence", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensorflow_session_from_frozen_graph_calculator_test", - srcs = ["tensorflow_session_from_frozen_graph_calculator_test.cc"], - data = [":test_frozen_graph"], - linkstatic = 1, - deps = [ - ":tensorflow_inference_calculator", - ":tensorflow_session", - ":tensorflow_session_from_frozen_graph_calculator", - ":tensorflow_session_from_frozen_graph_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map_helper", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:testlib", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core/kernels:math", - ], -) - -cc_test( - name = "tensorflow_session_from_frozen_graph_generator_test", - srcs = ["tensorflow_session_from_frozen_graph_generator_test.cc"], - data = [":test_frozen_graph"], - linkstatic = 1, - deps = [ - ":tensorflow_inference_calculator", - ":tensorflow_session", - ":tensorflow_session_from_frozen_graph_generator", - ":tensorflow_session_from_frozen_graph_generator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map_helper", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:testlib", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core/kernels:math", - ], -) - -cc_test( - name = "tensorflow_session_from_saved_model_generator_test", - srcs = ["tensorflow_session_from_saved_model_generator_test.cc"], - data = [":test_saved_model"], - linkstatic = 1, - deps = [ - ":tensorflow_inference_calculator", - ":tensorflow_session", - ":tensorflow_session_from_saved_model_generator", - ":tensorflow_session_from_saved_model_generator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:tag_map_helper", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:all_kernels", - "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensorflow_session_from_saved_model_calculator_test", - srcs = ["tensorflow_session_from_saved_model_calculator_test.cc"], - data = [":test_saved_model"], - linkstatic = 1, - deps = [ - ":tensorflow_inference_calculator", - ":tensorflow_session", - ":tensorflow_session_from_saved_model_calculator", - ":tensorflow_session_from_saved_model_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:tag_map_helper", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:all_kernels", - "@org_tensorflow//tensorflow/core:direct_session", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensor_squeeze_dimensions_calculator_test", - srcs = ["tensor_squeeze_dimensions_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":tensor_squeeze_dimensions_calculator", - ":tensor_squeeze_dimensions_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensor_to_image_frame_calculator_test", - size = "small", - srcs = ["tensor_to_image_frame_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":tensor_to_image_frame_calculator", - ":tensor_to_image_frame_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensor_to_matrix_calculator_test", - size = "small", - srcs = ["tensor_to_matrix_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":tensor_to_matrix_calculator", - ":tensor_to_matrix_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "tensor_to_vector_float_calculator_test", - srcs = ["tensor_to_vector_float_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":tensor_to_vector_float_calculator", - ":tensor_to_vector_float_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "unpack_media_sequence_calculator_test", - srcs = ["unpack_media_sequence_calculator_test.cc"], - deps = [ - ":unpack_media_sequence_calculator", - "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:unpack_media_sequence_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:rectangle", - "//mediapipe/util:audio_decoder_cc_proto", - "//mediapipe/util/sequence:media_sequence", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "vector_int_to_tensor_calculator_test", - srcs = ["vector_int_to_tensor_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":vector_int_to_tensor_calculator", - ":vector_int_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "vector_float_to_tensor_calculator_test", - srcs = ["vector_float_to_tensor_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":vector_float_to_tensor_calculator", - ":vector_float_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -cc_test( - name = "vector_string_to_tensor_calculator_test", - srcs = ["vector_string_to_tensor_calculator_test.cc"], - deps = [ - ":vector_string_to_tensor_calculator", - ":vector_string_to_tensor_calculator_options_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all_cc", - ], -) - -test_suite( - name = "ios", - tags = ["ios"], -) - -test_suite( - name = "android", - tags = ["android"], -) - -cc_test( - name = "tensorflow_inference_calculator_test", - size = "medium", - srcs = ["tensorflow_inference_calculator_test.cc"], - data = [":test_frozen_graph"], - linkstatic = 1, - deps = [ - ":tensorflow_session", - ":tensorflow_inference_calculator", - ":tensorflow_session_from_frozen_graph_generator", - ":tensorflow_session_from_frozen_graph_generator_cc_proto", - "@com_google_absl//absl/flags:flag", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool:validate_type", - "//mediapipe/framework/port:gtest_main", - ] + select({ - "//conditions:default": [ - "@org_tensorflow//tensorflow/core:testlib", - "@org_tensorflow//tensorflow/core/kernels:math", - "@org_tensorflow//tensorflow/core/kernels:conv_ops", - "@org_tensorflow//tensorflow/core:direct_session", - ], - "//mediapipe:android": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_lib", - "@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib", - ], - "//mediapipe:ios": [ - "@org_tensorflow//tensorflow/core:portable_tensorflow_test_lib", - ], - }), -) - -filegroup( - name = "test_frozen_graph", - srcs = [ - "testdata/bundle/00000000/checkpoint", - "testdata/bundle/00000000/export.meta", - "testdata/bundle/00000000/export-00000-of-00001", - "testdata/frozen_graph_def.pb", - "testdata/model.chkpt-0", - "testdata/model.chkpt-0.meta", - "testdata/tf_graph_def.pb", - ], -) - -filegroup( - name = "test_saved_model", - srcs = [ - "testdata/tensorflow_saved_model/00000000/saved_model.pb", - "testdata/tensorflow_saved_model/00000000/variables/variables.data-00000-of-00001", - "testdata/tensorflow_saved_model/00000000/variables/variables.index", - ], -) diff --git a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.cc b/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.cc deleted file mode 100644 index 310d412bf..000000000 --- a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Generates row tensors of prescribed length that are initialized to zeros. -// The tensors are placed in an ordered map, which maps the tensors to the -// tensor tags, and emitted as a packet. This generator has been developed -// primarily to generate initialization states for LSTMs. - -#include -#include - -#include "mediapipe/calculators/tensorflow/graph_tensors_packet_generator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/status_util.h" -#include "tensorflow/core/framework/tensor.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; - -class GraphTensorsPacketGenerator : public PacketGenerator { - public: - static absl::Status FillExpectations( - const PacketGeneratorOptions& extendable_options, - PacketTypeSet* input_side_packets, PacketTypeSet* output_side_packets) { - RET_CHECK(extendable_options.HasExtension( - GraphTensorsPacketGeneratorOptions::ext)); - const auto& options = extendable_options.GetExtension( // NOLINT - GraphTensorsPacketGeneratorOptions::ext); - output_side_packets->Index(0) - .Set>>( - /* "A map of tensor tags and tensors" */); - RET_CHECK_EQ(options.tensor_tag_size(), options.tensor_num_nodes_size()); - RET_CHECK_GT(options.tensor_tag_size(), 0); - return absl::OkStatus(); - } - - static absl::Status Generate( - const PacketGeneratorOptions& packet_generator_options, - const PacketSet& input_side_packets, PacketSet* output_side_packets) { - const GraphTensorsPacketGeneratorOptions& options = - packet_generator_options.GetExtension( - GraphTensorsPacketGeneratorOptions::ext); - // Output bundle packet. - auto tensor_map = absl::make_unique>(); - - for (int i = 0; i < options.tensor_tag_size(); ++i) { - const std::string& tensor_tag = options.tensor_tag(i); - const int32 tensor_num_nodes = options.tensor_num_nodes(i); - (*tensor_map)[tensor_tag] = - tf::Tensor(tf::DT_FLOAT, tf::TensorShape{1, tensor_num_nodes}); - (*tensor_map)[tensor_tag].flat().setZero(); - } - output_side_packets->Index(0) = AdoptAsUniquePtr(tensor_map.release()); - return absl::OkStatus(); - } -}; -REGISTER_PACKET_GENERATOR(GraphTensorsPacketGenerator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.proto b/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.proto deleted file mode 100644 index 1196ca1ef..000000000 --- a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/packet_generator.proto"; - -message GraphTensorsPacketGeneratorOptions { - extend mediapipe.PacketGeneratorOptions { - optional GraphTensorsPacketGeneratorOptions ext = 142721046; - } - - // Names of tensor tags for each of the generated tensors. - // Examples are: "STATE_C_0" or "STATE_M_0". - repeated string tensor_tag = 1; - - // Must be same length as tensor_tag. Each tensor tag must be paired with the - // number of nodes. - // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "STATE_C_0" - // and "STATE_M_0". Then, those tags can be used as the MediaPipe tags of - // tensors to initialized in TensorflowInferenceCalculator consuming - // the packet produced by this generator. For example, a mediapipe graph - // can include the node: - // packet_generator { - // packet_generator: "GraphTensorsPacketGenerator" - // output_side_packet: "init_tensors" - // options { - // [mediapipe.StateTensorsPacketGeneratorOptions.ext]: { - // tensor_tag: "STATE_C_0" - // tensor_num_nodes:128 - // tensor_tag: "STATE_M_0" - // tensor_num_nodes:128 - // } - // } - // } - repeated int32 tensor_num_nodes = 2; -} diff --git a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator_test.cc b/mediapipe/calculators/tensorflow/graph_tensors_packet_generator_test.cc deleted file mode 100644 index ef77fb918..000000000 --- a/mediapipe/calculators/tensorflow/graph_tensors_packet_generator_test.cc +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/graph_tensors_packet_generator.pb.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/packet_generator.pb.h" -#include "mediapipe/framework/packet_set.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/tensor.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -// Helper function that creates a row tensor that is initialized to zeros. -tf::Tensor ZeroRowTensor(const int col_length) { - tf::Tensor tensor(tf::DT_FLOAT, tf::TensorShape{1, col_length}); - tensor.flat().setZero(); - - return tensor; -} - -class GraphTensorsPacketGeneratorTest : public ::testing::Test { - protected: - void SetUp() override { - extendable_options_.Clear(); - generator_options_ = extendable_options_.MutableExtension( - GraphTensorsPacketGeneratorOptions::ext); - generator_options_->add_tensor_tag("A"); - generator_options_->add_tensor_num_nodes(3); - generator_options_->add_tensor_tag("B"); - generator_options_->add_tensor_num_nodes(4); - } - - void VerifyTensorMap(PacketSet* output_side_packets) { - const std::map* tensor_map = - GetFromUniquePtr>( - output_side_packets->Index(0)); - - EXPECT_FALSE(tensor_map->find("A") == tensor_map->end()); - EXPECT_FALSE(tensor_map->find("B") == tensor_map->end()); - - tf::Tensor expected_tensor = ZeroRowTensor(3); - EXPECT_EQ(expected_tensor.DebugString(), - tensor_map->find("A")->second.DebugString()); - - expected_tensor = ZeroRowTensor(4); - EXPECT_EQ(expected_tensor.DebugString(), - tensor_map->find("B")->second.DebugString()); - } - PacketGeneratorOptions extendable_options_; - GraphTensorsPacketGeneratorOptions* generator_options_; -}; - -// Test that the tensors are of the right size and shape -TEST_F(GraphTensorsPacketGeneratorTest, VerifyTensorSizeShapeAndValue) { - PacketSet inputs({}); - PacketSet outputs(1); - - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "GraphTensorsPacketGenerator", extendable_options_, inputs, &outputs); - MP_EXPECT_OK(run_status) << run_status.message(); - VerifyTensorMap(&outputs); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.cc deleted file mode 100644 index 0db193bcc..000000000 --- a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.cc +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace tf = tensorflow; - -namespace { -// Convert the ImageFrame into Tensor with floating point value type. -// The value will be normalized based on mean and stddev. -std::unique_ptr ImageFrameToNormalizedTensor( - const ImageFrame& image_frame, float mean, float stddev) { - const int cols = image_frame.Width(); - const int rows = image_frame.Height(); - const int channels = image_frame.NumberOfChannels(); - const uint8* pixel = image_frame.PixelData(); - const int width_padding = image_frame.WidthStep() - cols * channels; - auto tensor = ::absl::make_unique( - tf::DT_FLOAT, tf::TensorShape({rows, cols, channels})); - auto tensor_data = tensor->tensor(); - - for (int row = 0; row < rows; ++row) { - for (int col = 0; col < cols; ++col) { - for (int channel = 0; channel < channels; ++channel) { - tensor_data(row, col, channel) = (pixel[channel] - mean) / stddev; - } - pixel += channels; - } - pixel += width_padding; - } - return tensor; -} - -} // namespace - -// Converts ImageFrames to TensorFlow Tensors. -// -// The calculator expects one input (a packet containing an ImageFrame) and -// generates one output (a packet containing a tf::Tensor holding the same -// pixel data). The output tensor will be 3D with dimensions corresponding to -// height, width, and the number of channels (e.g. 3 for RGB or 1 for GRAY8). -// -// This calculator supports ImageFrame objects with any valid format (SRGB -// SRGBA, GRAY8, GRAY16, and VEC32F1). It will generate a Tensor using DT_UINT8 -// for the first three types, DT_UINT16 for GRAY16, and DT_FLOAT for VEC32F1. -// -// The ImageFrame data can be packed or padded. The pixel data will be copied -// to the Tensor in row-major order. -// -// Example config: -// node { -// calculator: "ImageFrameToTensorCalculator" -// input_stream: "scaled_frames" -// output_stream: "video_tensors" -// } -class ImageFrameToTensorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - ImageFrameToTensorCalculatorOptions options_; -}; -REGISTER_CALCULATOR(ImageFrameToTensorCalculator); - -absl::Status ImageFrameToTensorCalculator::GetContract(CalculatorContract* cc) { - // Start with only one input packet. - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - cc->Inputs().Index(0).Set( - // ImageFrame frame. - ); - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Index(0).Set( - // Output TensorFlow Tensor. - ); - return absl::OkStatus(); -} - -absl::Status ImageFrameToTensorCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - // Inform the framework that we always output at the same timestamp - // as we receive a packet at. - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status ImageFrameToTensorCalculator::Process(CalculatorContext* cc) { - const Packet& input_item = cc->Inputs().Index(0).Value(); - RET_CHECK(!input_item.IsEmpty()) << "Input cannot be empty."; - - // Extract the ImageFrame and metadata from the input packet. - const ImageFrame& video_frame = input_item.Get(); - const int bytes_per_pixel = video_frame.ByteDepth(); - - std::unique_ptr tensor; - if (options_.has_data_type()) { - RET_CHECK_EQ(bytes_per_pixel, 1) << "Unsupported image format (" - << bytes_per_pixel << " bytes per pixel)"; - const tf::DataType data_type = options_.data_type(); - RET_CHECK_EQ(data_type, tf::DT_FLOAT) - << "Unsupported data type " << data_type; - RET_CHECK_GT(options_.stddev(), 0.0f); - tensor = ImageFrameToNormalizedTensor(video_frame, options_.mean(), - options_.stddev()); - } else { - const int height = video_frame.Height(); - const int width = video_frame.Width(); - const int num_channels = video_frame.NumberOfChannels(); - const int num_components = width * height * num_channels; - tf::TensorShape tensor_shape({height, width, num_channels}); - - // Use uint8 uint16, or float as the TF type depending on bpp of ImageFrame. - tf::DataType data_type; - if (bytes_per_pixel == 1) { - data_type = tf::DT_UINT8; - } else if (bytes_per_pixel == 2) { - data_type = tf::DT_UINT16; - } else if (bytes_per_pixel == 4) { - data_type = tf::DT_FLOAT; - } else { - return absl::InvalidArgumentError(absl::StrCat( - "Unsupported image format (", bytes_per_pixel, " bytes per pixel)")); - } - - // This failure should never trigger, but it protects the code against - // internal TF changes. - RET_CHECK(tf::DataTypeCanUseMemcpy(data_type)) - << "Tensor data type does not support memcpy (type=" << data_type - << ")"; - - // Create the output tensor. - tensor = ::absl::make_unique(data_type, tensor_shape); - - // Copy pixel data from the ImageFrame to the tensor. - if (data_type == tf::DT_UINT8) { - uint8* dst = tensor->flat().data(); - video_frame.CopyToBuffer(dst, num_components); - } else if (data_type == tf::DT_UINT16) { - uint16* dst = tensor->flat().data(); - video_frame.CopyToBuffer(dst, num_components); - } else { - float* dst = tensor->flat().data(); - video_frame.CopyToBuffer(dst, num_components); - } - } - - cc->Outputs().Index(0).Add(tensor.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.proto b/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.proto deleted file mode 100644 index 0e5a47716..000000000 --- a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "tensorflow/core/framework/types.proto"; - -message ImageFrameToTensorCalculatorOptions { - extend CalculatorOptions { - optional ImageFrameToTensorCalculatorOptions ext = 120603667; - } - - // If set, the output tensor will be of data type specified by this field. - // Otherwise, the output tensor data type is equal to that of the input image - // frame. - optional tensorflow.DataType data_type = 1; - - // If set, the output tensor T is equal to (F - mean * J) / stddev, where F - // and J are the input image frame and the all-ones matrix of the same size, - // respectively. Otherwise, T is equal to F. - optional float mean = 2; - optional float stddev = 3; -} diff --git a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator_test.cc b/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator_test.cc deleted file mode 100644 index 86b12d6f9..000000000 --- a/mediapipe/calculators/tensorflow/image_frame_to_tensor_calculator_test.cc +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace tf = tensorflow; -using RandomEngine = std::mt19937_64; - -const uint8 kGray8 = 42; -const uint16 kGray16 = 4242; -const float kFloat = 42.0; -const uint kRed = 255; -const uint kGreen = 36; -const uint kBlue = 156; -const uint kAlpha = 42; - -const int kFixedNoiseWidth = 3; -const int kFixedNoiseHeight = 2; -const uint8 kFixedNoiseData[kFixedNoiseWidth * kFixedNoiseHeight * 3] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 213, 156, 9, 10, 11, 255, 0, 128}; - -class ImageFrameToTensorCalculatorTest : public ::testing::Test { - protected: - // Set image_frame to a constant per-channel pix_value. - template - void SetToColor(const T* pix_value, ImageFrame* image_frame) { - const int cols = image_frame->Width(); - const int rows = image_frame->Height(); - const int channels = image_frame->NumberOfChannels(); - const int width_padding = - image_frame->WidthStep() / (sizeof(T)) - cols * channels; - T* pixel = reinterpret_cast(image_frame->MutablePixelData()); - for (int row = 0; row < rows; ++row) { - for (int col = 0; col < cols; ++col) { - for (int channel = 0; channel < channels; ++channel) { - pixel[channel] = pix_value[channel]; - } - pixel += channels; - } - pixel += width_padding; - } - } - - // Adds a packet with a solid red 8-bit RGB ImageFrame. - void AddRGBFrame(int width, int height) { - auto image_frame = - ::absl::make_unique(ImageFormat::SRGB, width, height); - const uint8 color[] = {kRed, kGreen, kBlue}; - SetToColor(color, image_frame.get()); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with a solid red 8-bit RGBA ImageFrame. - void AddRGBAFrame(int width, int height) { - auto image_frame = - ::absl::make_unique(ImageFormat::SRGBA, width, height); - const uint8 color[] = {kRed, kGreen, kBlue, kAlpha}; - SetToColor(color, image_frame.get()); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with a solid GRAY8 ImageFrame. - void AddGray8Frame(int width, int height) { - auto image_frame = - ::absl::make_unique(ImageFormat::GRAY8, width, height); - const uint8 gray[] = {kGray8}; - SetToColor(gray, image_frame.get()); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with a solid GRAY16 ImageFrame. - void AddGray16Frame(int width, int height) { - auto image_frame = - ::absl::make_unique(ImageFormat::GRAY16, width, height, 1); - const uint16 gray[] = {kGray16}; - SetToColor(gray, image_frame.get()); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with a solid VEC32F1 ImageFrame. - void AddFloatFrame(int width, int height) { - auto image_frame = - ::absl::make_unique(ImageFormat::VEC32F1, width, height, 1); - const float gray[] = {kFloat}; - SetToColor(gray, image_frame.get()); - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with an 8-bit RGB ImageFrame containing pre-determined noise. - void AddFixedNoiseRGBFrame() { - auto image_frame = ::absl::make_unique( - ImageFormat::SRGB, kFixedNoiseWidth, kFixedNoiseHeight); - - // Copy fixed noise data into the ImageFrame. - const uint8* src = kFixedNoiseData; - uint8* pixels = image_frame->MutablePixelData(); - for (int y = 0; y < kFixedNoiseHeight; ++y) { - uint8* row = pixels + y * image_frame->WidthStep(); - std::memcpy(row, src, kFixedNoiseWidth * 3); - src += kFixedNoiseWidth * 3; - } - - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - // Adds a packet with an 8-bit RGB ImageFrame containing random noise. - void AddRandomRGBFrame(int width, int height, uint32 seed) { - RandomEngine random(seed); - std::uniform_int_distribution uniform_dist{ - 0, std::numeric_limits::max()}; - auto image_frame = - ::absl::make_unique(ImageFormat::SRGB, width, height); - - // Copy "noisy data" into the ImageFrame. - const int num_components_per_row = width * image_frame->NumberOfChannels(); - uint8* pixels = image_frame->MutablePixelData(); - for (int y = 0; y < kFixedNoiseHeight; ++y) { - uint8* p = pixels + y * image_frame->WidthStep(); - for (int i = 0; i < num_components_per_row; ++i) { - p[i] = uniform_dist(random); - } - } - - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - } - - std::unique_ptr runner_; -}; - -TEST_F(ImageFrameToTensorCalculatorTest, SolidRedRGBFrame) { - // Check two widths to cover packed and padded ImageFrame. - const int num_widths = 2; - const int widths[num_widths] = {10, 24}; - const int height = 5; - for (int width_index = 0; width_index < num_widths; ++width_index) { - const int width = widths[width_index]; - const int num_pixels = width * height; - - // Run the calculator and verify that one output is generated. - runner_ = ::absl::make_unique( - "ImageFrameToTensorCalculator", "", 1, 1, 0); - AddRGBFrame(width, height); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT8, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(3, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const uint8* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kRed, pixels[0]); - ASSERT_EQ(kGreen, pixels[1]); - ASSERT_EQ(kBlue, pixels[2]); - pixels += 3; - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, SolidRedRGBAFrame) { - // Check two widths to cover packed and padded ImageFrame. - const int num_widths = 2; - const int widths[num_widths] = {10, 24}; - const int height = 5; - for (int width_index = 0; width_index < num_widths; ++width_index) { - const int width = widths[width_index]; - const int num_pixels = width * height; - - // Run the calculator and verify that one output is generated. - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddRGBAFrame(width, height); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT8, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(4, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const uint8* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kRed, pixels[0]); - ASSERT_EQ(kGreen, pixels[1]); - ASSERT_EQ(kBlue, pixels[2]); - ASSERT_EQ(kAlpha, pixels[3]); - pixels += 4; - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, SolidGray8Frame) { - // Check two widths to cover packed and padded ImageFrame. - const int num_widths = 2; - const int widths[num_widths] = {10, 24}; - const int height = 5; - for (int width_index = 0; width_index < num_widths; ++width_index) { - const int width = widths[width_index]; - const int num_pixels = width * height; - - // Run the calculator and verify that one output is generated. - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddGray8Frame(width, height); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT8, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(1, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const uint8* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kGray8, pixels[0]); - ++pixels; - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, SolidGray16Frame) { - // Check two widths to cover packed and padded ImageFrame. - const int num_widths = 2; - const int widths[num_widths] = {10, 24}; - const int height = 5; - for (int width_index = 0; width_index < num_widths; ++width_index) { - const int width = widths[width_index]; - const int num_pixels = width * height; - - // Run the calculator and verify that one output is generated. - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddGray16Frame(width, height); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT16, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(1, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const uint16* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kGray16, pixels[0]); - ++pixels; - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, SolidFloatFrame) { - // Check two widths to cover packed and padded ImageFrame. - const int num_widths = 2; - const int widths[num_widths] = {10, 24}; - const int height = 5; - for (int width_index = 0; width_index < num_widths; ++width_index) { - const int width = widths[width_index]; - const int num_pixels = width * height; - - // Run the calculator and verify that one output is generated. - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddFloatFrame(width, height); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_FLOAT, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(1, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const float* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kFloat, pixels[0]); - ++pixels; - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, FixedNoiseRGBFrame) { - // Run the calculator and verify that one output is generated. - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddFixedNoiseRGBFrame(); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT8, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(kFixedNoiseHeight, shape.dim_size(0)); - ASSERT_EQ(kFixedNoiseWidth, shape.dim_size(1)); - ASSERT_EQ(3, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - const int num_pixels = kFixedNoiseWidth * kFixedNoiseHeight; - const uint8* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - ASSERT_EQ(kFixedNoiseData[i], pixels[i]); - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, RandomRGBFrame) { - // Run the calculator and verify that one output is generated. - const uint32 seed = 1234; - const int height = 2; - for (int width = 1; width <= 33; ++width) { - runner_.reset( - new CalculatorRunner("ImageFrameToTensorCalculator", "", 1, 1, 0)); - AddRandomRGBFrame(width, height, seed); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that tensor is 3-dimensional - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_UINT8, tensor.dtype()); - - // Verify that each dimension has the correct size / number of channels. - const tf::TensorShape& shape = tensor.shape(); - ASSERT_EQ(height, shape.dim_size(0)); - ASSERT_EQ(width, shape.dim_size(1)); - ASSERT_EQ(3, shape.dim_size(2)); - - // Verify that the data in the tensor is correct. - RandomEngine random(seed); - std::uniform_int_distribution uniform_dist{ - 0, std::numeric_limits::max()}; - const int num_pixels = width * height; - const uint8* pixels = - reinterpret_cast(tensor.tensor_data().data()); - for (int i = 0; i < num_pixels; ++i) { - const uint8 expected = uniform_dist(random); - ASSERT_EQ(expected, pixels[i]); - } - } -} - -TEST_F(ImageFrameToTensorCalculatorTest, FixedRGBFrameWithMeanAndStddev) { - runner_ = ::absl::make_unique( - "ImageFrameToTensorCalculator", - "[mediapipe.ImageFrameToTensorCalculatorOptions.ext]" - "{data_type:DT_FLOAT mean:128.0 stddev:128.0}", - 1, 1, 0); - - // Create a single pixel image of fixed color #0080ff. - auto image_frame = ::absl::make_unique(ImageFormat::SRGB, 1, 1); - const uint8 color[] = {0, 128, 255}; - SetToColor(color, image_frame.get()); - - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(image_frame.release()).At(Timestamp(0))); - MP_ASSERT_OK(runner_->Run()); - - const auto& tensor = runner_->Outputs().Index(0).packets[0].Get(); - EXPECT_EQ(tensor.dtype(), tf::DT_FLOAT); - ASSERT_EQ(tensor.dims(), 3); - EXPECT_EQ(tensor.shape().dim_size(0), 1); - EXPECT_EQ(tensor.shape().dim_size(1), 1); - EXPECT_EQ(tensor.shape().dim_size(2), 3); - const float* actual = tensor.flat().data(); - EXPECT_EQ(actual[0], -1.0f); // ( 0 - 128) / 128 - EXPECT_EQ(actual[1], 0.0f); // (128 - 128) / 128 - EXPECT_EQ(actual[2], 127.0f / 128.0f); // (255 - 128) / 128 -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc b/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc deleted file mode 100644 index a07b95ccc..000000000 --- a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/profiler/circular_buffer.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_util.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/status.h" - -namespace mediapipe { - -const char kBufferSize[] = "BUFFER_SIZE"; -const char kOverlap[] = "OVERLAP"; -const char kTimestampOffset[] = "TIMESTAMP_OFFSET"; -const char kCalculatorOptions[] = "CALCULATOR_OPTIONS"; - -namespace tf = tensorflow; - -// Given an input stream of tensors, concatenates the tensors over timesteps. -// The concatenated output tensors can be specified to have overlap between -// output timesteps. The tensors are concatenated along the first dimension, and -// a flag controls whether a new first dimension is inserted before -// concatenation. -// -// The number of tensors output will be buffer_size less than the -// number of input tensors unless padding is set to a non-zero value in the -// options proto. -// -// The timestamp of the output batch will match the timestamp of the first -// tensor in that batch by default. (e.g. when buffer_size frames are added, the -// output tensor will have the timestamp of the first input.). This behavior can -// be adjusted by the timestamp_offset option. -// -// Example config without padding: -// node { -// calculator: "LappedTensorBufferCalculator" -// input_stream: "input_tensor" -// output_stream: "output_tensor" -// options { -// [mediapipe.LappedTensorBufferCalculatorOptions.ext] { -// buffer_size: 2 -// overlap: 1 -// add_batch_dim_to_tensors: false -// } -// } -// } -// -// Example config with padding and timestamp output: -// node { -// calculator: "LappedTensorBufferCalculator" -// input_stream: "input_tensor" -// output_stream: "output_tensor" -// output_stream: "output_timestamp" -// options { -// [mediapipe.LappedTensorBufferCalculatorOptions.ext] { -// buffer_size: 100 -// overlap: 50 -// add_batch_dim_to_tensors: true -// timestamp_offset: 25 -// padding: 25 -// } -// } -// } - -class LappedTensorBufferCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - // Adds a batch dimension to the input tensor if specified in the - // calculator options. - absl::Status AddBatchDimension(tf::Tensor* input_tensor); - // Sends the current buffer downstream. - absl::Status ProcessBuffer(CalculatorContext* cc); - - int steps_until_output_; - int buffer_size_; - int overlap_; - int timestamp_offset_; - int initialized_; - - std::unique_ptr> timestamp_buffer_; - std::unique_ptr> buffer_; - LappedTensorBufferCalculatorOptions options_; -}; - -REGISTER_CALCULATOR(LappedTensorBufferCalculator); - -absl::Status LappedTensorBufferCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - cc->Inputs().Index(0).Set( - // tensorflow::Tensor stream. - ); - RET_CHECK_LE(cc->Outputs().NumEntries(), 2) - << "Only one or two output stream(s) is/are supported."; - - if (cc->InputSidePackets().HasTag(kBufferSize)) { - cc->InputSidePackets().Tag(kBufferSize).Set(); - } - if (cc->InputSidePackets().HasTag(kOverlap)) { - cc->InputSidePackets().Tag(kOverlap).Set(); - } - if (cc->InputSidePackets().HasTag(kTimestampOffset)) { - cc->InputSidePackets().Tag(kTimestampOffset).Set(); - } - if (cc->InputSidePackets().HasTag(kCalculatorOptions)) { - cc->InputSidePackets() - .Tag(kCalculatorOptions) - .Set(); - } - cc->Outputs().Index(0).Set( - // Output tensorflow::Tensor stream with possibly overlapping steps. - ); - // Output timestamp stream with possibly overlapping steps. - if (cc->Outputs().NumEntries() > 1) { - cc->Outputs().Index(1).Set>(); - } - return absl::OkStatus(); -} - -absl::Status LappedTensorBufferCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - if (cc->InputSidePackets().HasTag(kCalculatorOptions)) { - options_ = cc->InputSidePackets() - .Tag(kCalculatorOptions) - .Get(); - } - buffer_size_ = options_.buffer_size(); - if (cc->InputSidePackets().HasTag(kBufferSize)) { - buffer_size_ = cc->InputSidePackets().Tag(kBufferSize).Get(); - } - overlap_ = options_.overlap(); - if (cc->InputSidePackets().HasTag(kOverlap)) { - overlap_ = cc->InputSidePackets().Tag(kOverlap).Get(); - } - timestamp_offset_ = options_.timestamp_offset(); - if (cc->InputSidePackets().HasTag(kTimestampOffset)) { - timestamp_offset_ = cc->InputSidePackets().Tag(kTimestampOffset).Get(); - } - - RET_CHECK_LT(overlap_, buffer_size_); - RET_CHECK_GE(timestamp_offset_, 0) - << "Negative timestamp_offset is not allowed."; - RET_CHECK_LT(timestamp_offset_, buffer_size_) - << "output_frame_num_offset has to be less than buffer_size."; - RET_CHECK_LT(options_.padding(), buffer_size_) - << "padding option must be smaller than buffer size."; - timestamp_buffer_ = - absl::make_unique>(buffer_size_); - buffer_ = absl::make_unique>(buffer_size_); - steps_until_output_ = buffer_size_ - options_.padding(); - initialized_ = false; - return absl::OkStatus(); -} - -absl::Status LappedTensorBufferCalculator::Process(CalculatorContext* cc) { - // These are cheap, shallow copies. - tensorflow::Tensor input_tensor( - cc->Inputs().Index(0).Get()); - if (options_.add_batch_dim_to_tensors()) { - RET_CHECK_OK(AddBatchDimension(&input_tensor)); - } - // Pad frames at the beginning with the first frame. - if (!initialized_) { - for (int i = 0; i < options_.padding(); ++i) { - buffer_->push_back(input_tensor); - timestamp_buffer_->push_back(cc->InputTimestamp()); - } - initialized_ = true; - } - buffer_->push_back(input_tensor); - timestamp_buffer_->push_back(cc->InputTimestamp()); - --steps_until_output_; - if (steps_until_output_ <= 0) { - MP_RETURN_IF_ERROR(ProcessBuffer(cc)); - } - - return absl::OkStatus(); -} - -absl::Status LappedTensorBufferCalculator::Close(CalculatorContext* cc) { - if (!initialized_ || options_.padding() == 0) { - return absl::OkStatus(); - } - int last_frame = buffer_size_ - steps_until_output_ - 1; - const auto& pad_frame = buffer_->Get(last_frame); - for (int i = 0; i < steps_until_output_ + options_.padding(); ++i) { - buffer_->push_back(pad_frame); - timestamp_buffer_->push_back(cc->InputTimestamp()); - } - MP_RETURN_IF_ERROR(ProcessBuffer(cc)); - - return absl::OkStatus(); -} - -// Adds a batch dimension to the input tensor if specified in the calculator -// options. -absl::Status LappedTensorBufferCalculator::AddBatchDimension( - tf::Tensor* input_tensor) { - if (options_.add_batch_dim_to_tensors()) { - tf::TensorShape new_shape(input_tensor->shape()); - new_shape.InsertDim(0, 1); - RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape)) - << "Could not add 0th dimension to tensor without changing its shape." - << " Current shape: " << input_tensor->shape().DebugString(); - } - return absl::OkStatus(); -} - -// Process buffer -absl::Status LappedTensorBufferCalculator::ProcessBuffer( - CalculatorContext* cc) { - auto concatenated = ::absl::make_unique(); - const tf::Status concat_status = tf::tensor::Concat( - std::vector(buffer_->begin(), buffer_->end()), - concatenated.get()); - RET_CHECK(concat_status.ok()) << concat_status.ToString(); - // Output cancatenated tensor. - cc->Outputs().Index(0).Add(concatenated.release(), - timestamp_buffer_->Get(timestamp_offset_)); - if (cc->Outputs().NumEntries() > 1) { - auto output_timestamp = ::absl::make_unique>(); - // Output timestamp vector. - *output_timestamp = std::vector(timestamp_buffer_->begin(), - timestamp_buffer_->end()); - RET_CHECK_EQ(output_timestamp->size(), buffer_size_) - << "Output timestamp size is not correct."; - cc->Outputs().Index(1).Add(output_timestamp.release(), - timestamp_buffer_->Get(timestamp_offset_)); - } - steps_until_output_ = buffer_size_ - overlap_; - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.proto b/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.proto deleted file mode 100644 index bcd14985b..000000000 --- a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.proto +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LappedTensorBufferCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional LappedTensorBufferCalculatorOptions ext = 175222228; - } - - // The output tensor will have this many tensors concatenated together along - // their first dimension. - optional int32 buffer_size = 1; - - // The overlap determines how many input tensors are shared between frames. - // Because the input tensors may have a non-singleton first dimension, this - // is not necessarily the number of overlapping entries in the first - // dimension. - optional int32 overlap = 2; - - // If true, inserts a singleton first dimension before concatenating the - // tensors together. - optional bool add_batch_dim_to_tensors = 3 [default = false]; - - // Timestamp offset for output batch. The valid range is [0, buffer_size). - // The timestamp of the output tensor will match the timestamp of the input - // correspeonding to the offset. For example, setting to 0 will output at the - // timestamp matching the first input tensor. Setting the timestamp_offset to - // int((N-1) / 2) output at the timestamp matching the middle input tensor. - // This is useful for aligning the timestamp to be centered on the input - // range. - optional int32 timestamp_offset = 4 [default = 0]; - - // Amount of padding (repeating of first/last value) to add to the beginning - // and end of the input stream. - optional int32 padding = 5; -} diff --git a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator_test.cc b/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator_test.cc deleted file mode 100644 index e0e3000d2..000000000 --- a/mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator_test.cc +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class LappedTensorBufferCalculatorTest : public ::testing::Test { - protected: - void SetUpCalculator(int buffer_size, int overlap, bool add_dim, - int timestamp_offset, int padding, - bool timestamp_output) { - CalculatorGraphConfig::Node config; - config.set_calculator("LappedTensorBufferCalculator"); - config.add_input_stream("input_tensor"); - config.add_output_stream("output_tensor"); - if (timestamp_output) { - config.add_output_stream("output_timestamp"); - } - auto options = config.mutable_options()->MutableExtension( - LappedTensorBufferCalculatorOptions::ext); - options->set_buffer_size(buffer_size); - options->set_overlap(overlap); - if (add_dim) { - options->set_add_batch_dim_to_tensors(true); - } - options->set_timestamp_offset(timestamp_offset); - options->set_padding(padding); - runner_ = ::absl::make_unique(config); - } - std::unique_ptr runner_; -}; - -TEST_F(LappedTensorBufferCalculatorTest, OneToOne) { - SetUpCalculator(1, 0, false, 0, 0, false); - int num_timesteps = 3; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps, output_packets.size()); - for (int i = 0; i < num_timesteps; ++i) { - float value = output_packets[i].Get().tensor()(0); - ASSERT_NEAR(i, value, 0.0001); - } -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToTwo) { - int buffer_size = 2; - int overlap = 1; - bool add_dim = false; - SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false); - int num_timesteps = 3; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps - buffer_size + 1, output_packets.size()); - for (int i = 0; i < num_timesteps - buffer_size + 1; ++i) { - for (int j = 0; j < buffer_size; ++j) { - float value = output_packets[i].Get().tensor()(j); - ASSERT_NEAR(i + j, value, 0.0001); - } - } -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToThree) { - int buffer_size = 3; - int overlap = 2; - bool add_dim = false; - SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false); - int num_timesteps = 3; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps - buffer_size + 1, output_packets.size()); - for (int i = 0; i < num_timesteps - buffer_size + 1; ++i) { - for (int j = 0; j < buffer_size; ++j) { - float value = output_packets[i].Get().tensor()(j); - ASSERT_NEAR(i + j, value, 0.0001); - } - } -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToThreeSkip) { - int buffer_size = 3; - int overlap = 1; - bool add_dim = false; - SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false); - int num_timesteps = 3; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps - buffer_size + 1, output_packets.size()); - for (int i = 0; i < num_timesteps - buffer_size + 1; ++i) { - for (int j = 0; j < buffer_size; ++j) { - float value = output_packets[i].Get().tensor()(j); - ASSERT_NEAR((i * 2) + j, value, 0.0001); - } - } -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToThreeNegativeOverlap) { - int buffer_size = 3; - int overlap = -1; - bool add_dim = false; - SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false); - int num_timesteps = 7; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(2, output_packets.size()); - // The outputs in packet one should be {0, 1, 2}, and in packet two {4, 5, 6} - for (int i = 0; i < 3; ++i) { - float value_0 = output_packets[0].Get().tensor()(i); - ASSERT_NEAR(value_0, i, 0.0001); - } - for (int i = 0; i < 3; ++i) { - float value_1 = output_packets[1].Get().tensor()(i); - ASSERT_NEAR(value_1, 4 + i, 0.0001); - } -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatch) { - int buffer_size = 3; - int overlap = 2; - bool add_dim = true; - SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false); - int num_timesteps = 3; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps - buffer_size + 1, output_packets.size()); - for (int i = 0; i < num_timesteps - buffer_size + 1; ++i) { - for (int j = 0; j < buffer_size; ++j) { - float value = - output_packets[i].Get().tensor()(j, 0); - ASSERT_NEAR(i + j, value, 0.0001); - } - } -} - -TEST_F(LappedTensorBufferCalculatorTest, NegativeTimestampOffsetFails) { - int buffer_size = 16; - int overlap = 15; - bool add_dim = true; - int timestamp_offset = -7; - SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false); - int num_timesteps = 20; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_FALSE(runner_->Run().ok()); -} - -TEST_F(LappedTensorBufferCalculatorTest, OutOfRangeTimestampOffsetFails) { - int buffer_size = 16; - int overlap = 15; - bool add_dim = true; - int timestamp_offset = buffer_size; - SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false); - int num_timesteps = 20; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_FALSE(runner_->Run().ok()); -} - -TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) { - int buffer_size = 16; - int overlap = 15; - bool add_dim = true; - int timestamp_offset = 7; - SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false); - int num_timesteps = 20; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(num_timesteps - buffer_size + 1, output_packets.size()); - for (int i = 0; i < num_timesteps - buffer_size + 1; ++i) { - for (int j = 0; j < buffer_size; ++j) { - int64 value = output_packets[i].Timestamp().Value(); - ASSERT_EQ(i + timestamp_offset, value); - } - } -} - -TEST_F(LappedTensorBufferCalculatorTest, - OneToThreeBatchTimestampOffsetPadding) { - int buffer_size = 12; - int overlap = 6; - bool add_dim = true; - int timestamp_offset = 3; - int padding = 0; - SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, padding, - true); - int num_timesteps = 20; - for (int i = 0; i < num_timesteps; ++i) { - auto input = ::absl::make_unique( - tensorflow::DT_FLOAT, tensorflow::TensorShape({1})); - input->tensor()(0) = i; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(i))); - } - ASSERT_TRUE(runner_->Run().ok()); - - const int output_size = num_timesteps / buffer_size + 1; - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(output_size, output_packets.size()); - for (int i = 0; i < output_size; ++i) { - int64 value = output_packets[i].Timestamp().Value(); - ASSERT_EQ(i * overlap + timestamp_offset, value); - } - const std::vector& output_timestamps = - runner_->Outputs().Index(1).packets; - ASSERT_EQ(output_size, output_timestamps.size()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc deleted file mode 100644 index 32a0eb70b..000000000 --- a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace { -absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, - TimeSeriesHeader* header) { - CHECK(header); - if (header_packet.IsEmpty()) { - return absl::UnknownError("No header found."); - } - if (!header_packet.ValidateAsType().ok()) { - return absl::UnknownError("Packet does not contain TimeSeriesHeader."); - } - *header = header_packet.Get(); - if (header->has_sample_rate() && header->sample_rate() >= 0 && - header->has_num_channels() && header->num_channels() >= 0) { - return absl::OkStatus(); - } else { - std::string error_message = - "TimeSeriesHeader is missing necessary fields: " - "sample_rate or num_channels, or one of their values is negative. "; -#ifndef MEDIAPIPE_MOBILE - absl::StrAppend(&error_message, "Got header:\n", - header->ShortDebugString()); -#endif - return absl::InvalidArgumentError(error_message); - } -} -} // namespace - -namespace tf = tensorflow; - -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - -// Converts an input Matrix into a 2D or 3D tf::Tensor. -// -// The calculator expects one input (a packet containing a Matrix) and -// generates one output (a packet containing a tf::Tensor containing the same -// data). The output tensor will be 2D with dimensions corresponding to the -// input matrix, while it will be 3D if add_trailing_dimension is set to true. -// The option for making the tensor be 3D is useful for using audio and image -// features for training multimodal models, so that the number of tensor -// dimensions match up. It will hold DT_FLOAT values. -// -// Example config: -// node { -// calculator: "MatrixToTensorCalculator" -// input_stream: "matrix_features" -// output_stream: "tensor_features" -// } -class MatrixToTensorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - MatrixToTensorCalculatorOptions options_; -}; -REGISTER_CALCULATOR(MatrixToTensorCalculator); - -absl::Status MatrixToTensorCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - cc->Inputs().Index(0).Set( - // Input Matrix stream with optional TimeSeriesHeader. - ); - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Index(0).Set( - // Output stream with data as tf::Tensor and the same - // TimeSeriesHeader as the input (or no header if the input has no - // header). - ); - return absl::OkStatus(); -} - -absl::Status MatrixToTensorCalculator::Open(CalculatorContext* cc) { - // If the input is part of a time series, then preserve the header so that - // downstream consumers can access the sample rate if needed. - options_ = cc->Options(); - auto input_header = ::absl::make_unique(); - const absl::Status header_status = FillTimeSeriesHeaderIfValid( - cc->Inputs().Index(0).Header(), input_header.get()); - if (header_status.ok()) { - cc->Outputs().Index(0).SetHeader(Adopt(input_header.release())); - } - - // Inform the framework that we always output at the same timestamp - // as we receive a packet at. - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status MatrixToTensorCalculator::Process(CalculatorContext* cc) { - const Matrix& matrix = cc->Inputs().Index(0).Get(); - tf::TensorShape tensor_shape; - if (options_.transpose()) { - tensor_shape = tf::TensorShape({matrix.cols(), matrix.rows()}); - } else { - tensor_shape = tf::TensorShape({matrix.rows(), matrix.cols()}); - } - auto tensor = ::absl::make_unique(tf::DT_FLOAT, tensor_shape); - - float* tensor_data = tensor->flat().data(); - if (options_.transpose()) { - auto matrix_map = - Eigen::Map(tensor_data, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_data, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - - if (options_.add_trailing_dimension()) { - tf::TensorShape new_shape(tensor_shape); - new_shape.AddDim(1 /* size of dimension */); - RET_CHECK(tensor->CopyFrom(*tensor, new_shape)) - << "Could not add dimension to tensor without changing its shape." - << " Current shape: " << tensor->shape().DebugString(); - } - cc->Outputs().Index(0).Add(tensor.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.proto b/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.proto deleted file mode 100644 index 50a1775cc..000000000 --- a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message MatrixToTensorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional MatrixToTensorCalculatorOptions ext = 130781699; - } - - optional bool transpose = 1 [default = false]; - // Adds a 3rd dimension of size 1 when this is set to true. - optional bool add_trailing_dimension = 2 [default = false]; -} diff --git a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc b/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc deleted file mode 100644 index c3c6ef5f8..000000000 --- a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_test.cc +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace { - -constexpr char kTransposeOptionsString[] = - "[mediapipe.MatrixToTensorCalculatorOptions.ext]: {" - "transpose: True}"; -constexpr char kAddDimensionOptionsString[] = - "[mediapipe.MatrixToTensorCalculatorOptions.ext]: {" - "add_trailing_dimension: True}"; - -} // namespace - -namespace tf = tensorflow; -using RandomEngine = std::mt19937_64; -const uint32 kSeed = 1234; -const int kNumSizes = 8; -const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, - {5, 3}, {7, 13}, {16, 32}, {101, 2}}; - -class MatrixToTensorCalculatorTest : public ::testing::Test { - protected: - // Adds a packet with a matrix filled with random values in [0,1]. - void AddRandomMatrix(int num_rows, int num_columns, uint32 seed) { - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - auto matrix = ::absl::make_unique(); - matrix->resize(num_rows, num_columns); - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - (*matrix)(y, x) = uniform_dist(random); - } - } - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(matrix.release()).At(Timestamp(0))); - } - - std::unique_ptr runner_; -}; - -TEST_F(MatrixToTensorCalculatorTest, RandomMatrix) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - runner_ = ::absl::make_unique("MatrixToTensorCalculator", - "", 1, 1, 0); - AddRandomMatrix(num_rows, num_columns, kSeed); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that the packet contains a 2D float tensor. - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(2, tensor.dims()); - ASSERT_EQ(tf::DT_FLOAT, tensor.dtype()); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - const auto matrix = tensor.matrix(); - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - const float expected = uniform_dist(random); - ASSERT_EQ(expected, matrix(y, x)); - } - } - } -} - -TEST_F(MatrixToTensorCalculatorTest, RandomMatrixTranspose) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - runner_ = ::absl::make_unique( - "MatrixToTensorCalculator", kTransposeOptionsString, 1, 1, 0); - AddRandomMatrix(num_rows, num_columns, kSeed); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that the packet contains a 2D float tensor. - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(2, tensor.dims()); - ASSERT_EQ(tf::DT_FLOAT, tensor.dtype()); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - const auto matrix = tensor.matrix(); - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - const float expected = uniform_dist(random); - ASSERT_EQ(expected, matrix(x, y)); - } - } - } -} - -TEST_F(MatrixToTensorCalculatorTest, RandomMatrixAddDimension) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - runner_ = ::absl::make_unique( - "MatrixToTensorCalculator", kAddDimensionOptionsString, 1, 1, 0); - AddRandomMatrix(num_rows, num_columns, kSeed); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - ASSERT_EQ(1, output_packets.size()); - - // Verify that the packet contains a 3D float tensor. - const tf::Tensor& tensor = output_packets[0].Get(); - ASSERT_EQ(3, tensor.dims()); - ASSERT_EQ(tf::DT_FLOAT, tensor.dtype()); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - // const auto matrix = tensor.matrix(); - const float* tensor_data = tensor.flat().data(); - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - const float expected = uniform_dist(random); - ASSERT_EQ(expected, tensor_data[y * num_columns + x]); - } - } - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.cc b/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.cc deleted file mode 100644 index a8abe10d9..000000000 --- a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.cc +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/source_location.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" -#include "mediapipe/util/tensor_to_detection.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" - -namespace mediapipe { -class CalculatorOptions; -} // namespace mediapipe - -namespace mediapipe { - -namespace tf = ::tensorflow; - -namespace { -const char kNumDetections[] = "NUM_DETECTIONS"; -const char kBoxes[] = "BOXES"; -const char kScores[] = "SCORES"; -const char kClasses[] = "CLASSES"; -const char kDetections[] = "DETECTIONS"; -const char kKeypoints[] = "KEYPOINTS"; -const char kMasks[] = "MASKS"; -const char kLabelMap[] = "LABELMAP"; -const int kNumCoordsPerBox = 4; -} // namespace - -// Takes object detection results and converts them into MediaPipe Detections. -// -// Inputs are assumed to be tensors of the form: -// `num_detections` : float32 scalar tensor indicating the number of valid -// detections. -// `detection_boxes` : float32 tensor of the form [num_boxes, 4]. Format for -// coordinates is {ymin, xmin, ymax, xmax}. -// `detection_scores` : float32 tensor of the form [num_boxes]. -// `detection_classes` : float32 tensor of the form [num_boxes]. -// `detection_keypoints`: float32 tensor of the form -// [num_boxes, num_keypoints, 2]. -// `detection_masks` : float32 tensor of the form -// [num_boxes, height, width]. -// -// These are generated according to the Vale object detector model exporter, -// which may be found in -// image/understanding/object_detection/export_inference_graph.py -// -// By default, the output Detections store label ids (integers) for each -// detection. Optionally, a label map (of the form std::map -// mapping label ids to label names as strings) can be made available as an -// input side packet, in which case the output Detections store -// labels as their associated std::string provided by the label map. -// -// Usage example: -// node { -// calculator: "ObjectDetectionTensorsToDetectionsCalculator" -// input_stream: "BOXES:detection_boxes_tensor" -// input_stream: "SCORES:detection_scores_tensor" -// input_stream: "CLASSES:detection_classes_tensor" -// input_stream: "NUM_DETECTIONS:num_detections_tensor" -// output_stream: "DETECTIONS:detections" -// options: { -// [mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions.ext]: { -// tensor_dim_to_squeeze: 0 -// } -// } -// } -class ObjectDetectionTensorsToDetectionsCalculator : public CalculatorBase { - public: - ObjectDetectionTensorsToDetectionsCalculator() = default; - - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kBoxes).Set(); - cc->Inputs().Tag(kScores).Set(); - - if (cc->Inputs().HasTag(kNumDetections)) { - cc->Inputs().Tag(kNumDetections).Set(); - } - if (cc->Inputs().HasTag(kClasses)) { - cc->Inputs().Tag(kClasses).Set(); - } - if (cc->Inputs().HasTag(kKeypoints)) { - cc->Inputs().Tag(kKeypoints).Set(); - } - - if (cc->Inputs().HasTag(kMasks)) { - cc->Inputs().Tag(kMasks).Set(); - - const auto& calculator_options = - cc->Options(); - float mask_threshold = calculator_options.mask_threshold(); - if (!(mask_threshold >= 0.0 && mask_threshold <= 1.0)) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "mask_threshold must be in range [0.0, 1.0]"; - } - } - - cc->Outputs().Tag(kDetections).Set>(); - - if (cc->InputSidePackets().HasTag(kLabelMap)) { - cc->InputSidePackets() - .Tag(kLabelMap) - .Set>>(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - if (cc->InputSidePackets().HasTag(kLabelMap)) { - label_map_ = GetFromUniquePtr>( - cc->InputSidePackets().Tag(kLabelMap)); - } - const auto& tensor_dim_to_squeeze_field = - cc->Options() - .tensor_dim_to_squeeze(); - tensor_dims_to_squeeze_ = std::vector( - tensor_dim_to_squeeze_field.begin(), tensor_dim_to_squeeze_field.end()); - std::sort(tensor_dims_to_squeeze_.rbegin(), tensor_dims_to_squeeze_.rend()); - cc->SetOffset(0); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const auto& options = - cc->Options(); - - tf::Tensor input_num_detections_tensor = - tf::Tensor(tf::DT_FLOAT, tf::TensorShape({0})); - if (cc->Inputs().HasTag(kClasses)) { - ASSIGN_OR_RETURN( - input_num_detections_tensor, - MaybeSqueezeDims(kNumDetections, - cc->Inputs().Tag(kNumDetections).Get())); - } - if (input_num_detections_tensor.dtype() != tf::DT_INT32) { - RET_CHECK_EQ(input_num_detections_tensor.dtype(), tf::DT_FLOAT); - } - - ASSIGN_OR_RETURN( - auto input_boxes_tensor, - MaybeSqueezeDims(kBoxes, cc->Inputs().Tag(kBoxes).Get())); - RET_CHECK_EQ(input_boxes_tensor.dtype(), tf::DT_FLOAT); - - ASSIGN_OR_RETURN( - auto input_scores_tensor, - MaybeSqueezeDims(kScores, cc->Inputs().Tag(kScores).Get())); - RET_CHECK_EQ(input_scores_tensor.dtype(), tf::DT_FLOAT); - - tf::Tensor input_classes_tensor = - tf::Tensor(tf::DT_FLOAT, tf::TensorShape({0})); - if (cc->Inputs().HasTag(kClasses)) { - ASSIGN_OR_RETURN( - input_classes_tensor, - MaybeSqueezeDims(kClasses, - cc->Inputs().Tag(kClasses).Get())); - } - RET_CHECK_EQ(input_classes_tensor.dtype(), tf::DT_FLOAT); - - auto output_detections = absl::make_unique>(); - - const tf::Tensor& input_keypoints_tensor = - cc->Inputs().HasTag(kKeypoints) - ? cc->Inputs().Tag(kKeypoints).Get() - : tf::Tensor(tf::DT_FLOAT, tf::TensorShape({0, 0, 0})); - - const tf::Tensor& input_masks_tensor = - cc->Inputs().HasTag(kMasks) - ? cc->Inputs().Tag(kMasks).Get() - : tf::Tensor(tf::DT_FLOAT, tf::TensorShape({0, 0, 0})); - RET_CHECK_EQ(input_masks_tensor.dtype(), tf::DT_FLOAT); - - const std::map label_map = - (label_map_ == nullptr) ? std::map{} : *label_map_; - - RET_CHECK_OK(TensorsToDetections( - input_num_detections_tensor, input_boxes_tensor, input_scores_tensor, - input_classes_tensor, input_keypoints_tensor, input_masks_tensor, - options.mask_threshold(), label_map, output_detections.get())); - - cc->Outputs() - .Tag(kDetections) - .Add(output_detections.release(), cc->InputTimestamp()); - - return absl::OkStatus(); - } - - private: - std::map* label_map_; - std::vector tensor_dims_to_squeeze_; - - absl::StatusOr MaybeSqueezeDims(const std::string& tensor_tag, - const tf::Tensor& input_tensor) { - if (tensor_dims_to_squeeze_.empty()) { - return input_tensor; - } - tf::TensorShape tensor_shape = input_tensor.shape(); - for (const int dim : tensor_dims_to_squeeze_) { - RET_CHECK_GT(tensor_shape.dims(), dim) - << "Dimension " << dim - << " does not exist in input tensor with num dimensions " - << input_tensor.dims() << " dims"; - RET_CHECK_EQ(tensor_shape.dim_size(dim), 1) - << "Cannot remove dimension " << dim << " with size " - << tensor_shape.dim_size(dim); - tensor_shape.RemoveDim(dim); - } - tf::Tensor output_tensor; - RET_CHECK(output_tensor.CopyFrom(input_tensor, tensor_shape)); - return std::move(output_tensor); - } -}; - -REGISTER_CALCULATOR(ObjectDetectionTensorsToDetectionsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.proto b/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.proto deleted file mode 100644 index 149144684..000000000 --- a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator.proto +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// The option proto for the ObjectDetectionsTensorToDetectionsCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message ObjectDetectionsTensorToDetectionsCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional ObjectDetectionsTensorToDetectionsCalculatorOptions ext = - 192676232; - } - - // The threshold used to compute the binary segmentation mask. - optional float mask_threshold = 1 [default = 0.0]; - - // The specific singleton dimensions to squeeze (remove). The calculator can - // only removes dimensions of size 1. - repeated int32 tensor_dim_to_squeeze = 2; -} diff --git a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc b/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc deleted file mode 100644 index c14802c87..000000000 --- a/mediapipe/calculators/tensorflow/object_detection_tensors_to_detections_calculator_test.cc +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; -namespace { -const char kNumDetections[] = "NUM_DETECTIONS"; -const char kBoxes[] = "BOXES"; -const char kScores[] = "SCORES"; -const char kClasses[] = "CLASSES"; -const char kKeypoints[] = "KEYPOINTS"; -const char kDetections[] = "DETECTIONS"; -const int kNumBoxes = 3; -const int kNumClasses = 4; -const int kNumCoordsPerBox = 4; -const int kNumKeypointsPerBox = 2; -const int kNumCoordsPerKeypoint = 2; - -class ObjectDetectionTensorsToDetectionsCalculatorTest - : public ::testing::Test { - protected: - void SetUp() override { SetUpInputs(); } - - void SetUpInputs() { - input_num_detections_ = tf::test::AsTensor({kNumBoxes}, {1}); - // {ymin, xmin, ymax, xmax} format. - input_boxes_ = - tf::test::AsTensor({0.1f, 0.2f, 0.3f, 0.4f, 0.1f, 0.2f, 0.3f, - 0.4f, 0.1f, 0.2f, 0.3f, 0.4f}, - {kNumBoxes, kNumCoordsPerBox}); - input_scores_ = tf::test::AsTensor({0.1f, 0.5f, 1.0f}, {kNumBoxes}); - input_scores_for_all_classes_ = - tf::test::AsTensor({0.0f, 0.1f, 0.05f, 0.02f, 0.0f, 0.1f, 0.5f, - 0.2f, 0.0f, 0.5f, 0.8f, 1.0f}, - {kNumBoxes, kNumClasses}); - input_classes_ = tf::test::AsTensor({1.0, 2.0, 3.0}, {kNumBoxes}); - input_keypoints_ = tf::test::AsTensor( - {0.6f, 0.5f, 0.6f, 0.5f, 0.6f, 0.5f, 0.6f, 0.5f, 0.6f, 0.5f, 0.6f, - 0.5f}, - {kNumBoxes, kNumKeypointsPerBox, kNumCoordsPerKeypoint}); - } - - void CreateNodeConfig(CalculatorGraphConfig::Node* node_config) const { - ASSERT_NE(nullptr, node_config); - *node_config = ParseTextProtoOrDie(R"pb( - calculator: "ObjectDetectionTensorsToDetectionsCalculator" - input_stream: "NUM_DETECTIONS:num_detections" - input_stream: "BOXES:boxes" - input_stream: "SCORES:scores" - input_stream: "CLASSES:classes" - output_stream: "DETECTIONS:detections" - )pb"); - } - - void CreateNodeConfigRawTensors( - CalculatorGraphConfig::Node* node_config) const { - ASSERT_NE(nullptr, node_config); - *node_config = ParseTextProtoOrDie(R"pb( - calculator: "ObjectDetectionTensorsToDetectionsCalculator" - input_stream: "BOXES:raw_detection_boxes" - input_stream: "SCORES:raw_detection_scores" - output_stream: "DETECTIONS:detections" - )pb"); - } - - void CreateNodeConfigWithKeypoints( - CalculatorGraphConfig::Node* node_config) const { - ASSERT_NE(nullptr, node_config); - *node_config = ParseTextProtoOrDie(R"pb( - calculator: "ObjectDetectionTensorsToDetectionsCalculator" - input_stream: "NUM_DETECTIONS:num_detections" - input_stream: "BOXES:boxes" - input_stream: "SCORES:scores" - input_stream: "CLASSES:classes" - input_stream: "KEYPOINTS:keypoints" - output_stream: "DETECTIONS:detections" - )pb"); - } - - void SetUpCalculatorRunner() { - CalculatorGraphConfig::Node node_config; - CreateNodeConfig(&node_config); - runner_ = absl::make_unique(node_config); - } - - void SetUpCalculatorRunnerRawTensors() { - CalculatorGraphConfig::Node node_config; - CreateNodeConfigRawTensors(&node_config); - runner_ = absl::make_unique(node_config); - } - - void SetUpCalculatorRunnerWithKeypoints() { - CalculatorGraphConfig::Node node_config; - CreateNodeConfigWithKeypoints(&node_config); - runner_ = absl::make_unique(node_config); - } - - void RunCalculator() { - SetUpCalculatorRunner(); - runner_->MutableInputs() - ->Tag(kNumDetections) - .packets.push_back( - PointToForeign(&input_num_detections_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kBoxes).packets.push_back( - PointToForeign(&input_boxes_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kScores).packets.push_back( - PointToForeign(&input_scores_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kClasses).packets.push_back( - PointToForeign(&input_classes_).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner_->Run()); - ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size()); - } - - void RunCalculatorRawTensors() { - SetUpCalculatorRunnerRawTensors(); - runner_->MutableInputs()->Tag(kBoxes).packets.push_back( - PointToForeign(&input_boxes_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kScores).packets.push_back( - PointToForeign(&input_scores_for_all_classes_) - .At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner_->Run()); - ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size()); - } - - void RunCalculatorWithKeypoints() { - SetUpCalculatorRunnerWithKeypoints(); - runner_->MutableInputs() - ->Tag(kNumDetections) - .packets.push_back( - PointToForeign(&input_num_detections_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kBoxes).packets.push_back( - PointToForeign(&input_boxes_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kScores).packets.push_back( - PointToForeign(&input_scores_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kClasses).packets.push_back( - PointToForeign(&input_classes_).At(Timestamp::PostStream())); - runner_->MutableInputs() - ->Tag(kKeypoints) - .packets.push_back( - PointToForeign(&input_keypoints_).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner_->Run()); - ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size()); - } - - void RunCalculatorWithTensorDimensionSqueezing() { - InsertExtraSingltonDim(&input_num_detections_); - InsertExtraSingltonDim(&input_boxes_); - InsertExtraSingltonDim(&input_scores_); - InsertExtraSingltonDim(&input_classes_); - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "ObjectDetectionTensorsToDetectionsCalculator" - input_stream: "NUM_DETECTIONS:num_detections" - input_stream: "BOXES:boxes" - input_stream: "SCORES:scores" - input_stream: "CLASSES:classes" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.ObjectDetectionsTensorToDetectionsCalculatorOptions - .ext]: { tensor_dim_to_squeeze: 0 } - } - )pb"); - runner_ = absl::make_unique(node_config); - runner_->MutableInputs() - ->Tag(kNumDetections) - .packets.push_back( - PointToForeign(&input_num_detections_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kBoxes).packets.push_back( - PointToForeign(&input_boxes_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kScores).packets.push_back( - PointToForeign(&input_scores_).At(Timestamp::PostStream())); - runner_->MutableInputs()->Tag(kClasses).packets.push_back( - PointToForeign(&input_classes_).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner_->Run()); - ASSERT_EQ(1, runner_->Outputs().Tag(kDetections).packets.size()); - } - - void InsertExtraSingltonDim(tf::Tensor* tensor) { - tf::TensorShape new_shape(tensor->shape()); - new_shape.InsertDim(0, 1); - ASSERT_TRUE(tensor->CopyFrom(*tensor, new_shape)); - } - - std::unique_ptr runner_; - - tf::Tensor input_num_detections_; - tf::Tensor input_boxes_; - tf::Tensor input_scores_; - tf::Tensor input_scores_for_all_classes_; - tf::Tensor input_classes_; - tf::Tensor input_keypoints_; -}; - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, OutputsDetections) { - RunCalculator(); - EXPECT_EQ(kNumBoxes, runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>() - .size()); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - OutputsDetectionsFromRawTensors) { - RunCalculatorRawTensors(); - EXPECT_EQ(kNumBoxes, runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>() - .size()); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - OutputsDetectionsWithKeypoints) { - RunCalculatorWithKeypoints(); - EXPECT_EQ(kNumBoxes, runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>() - .size()); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - OutputsDetectionsWithCorrectValues) { - RunCalculator(); - const std::vector detections = runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>(); - EXPECT_EQ(kNumBoxes, detections.size()); - for (const auto& detection : detections) { - LocationData::RelativeBoundingBox relative_bbox = - detection.location_data().relative_bounding_box(); - EXPECT_FLOAT_EQ(0.2, relative_bbox.xmin()); - EXPECT_FLOAT_EQ(0.1, relative_bbox.ymin()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.width()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.height()); - } - EXPECT_FLOAT_EQ(0.1f, detections[0].score(0)); - EXPECT_FLOAT_EQ(0.5f, detections[1].score(0)); - EXPECT_FLOAT_EQ(1.0f, detections[2].score(0)); - EXPECT_EQ(1, detections[0].label_id(0)); - EXPECT_EQ(2, detections[1].label_id(0)); - EXPECT_EQ(3, detections[2].label_id(0)); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - OutputsDetectionsFromRawTensorsWithCorrectValues) { - RunCalculatorRawTensors(); - const std::vector detections = runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>(); - EXPECT_EQ(kNumBoxes, detections.size()); - for (const auto& detection : detections) { - LocationData::RelativeBoundingBox relative_bbox = - detection.location_data().relative_bounding_box(); - EXPECT_FLOAT_EQ(0.2, relative_bbox.xmin()); - EXPECT_FLOAT_EQ(0.1, relative_bbox.ymin()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.width()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.height()); - } - EXPECT_FLOAT_EQ(0.1f, detections[0].score(0)); - EXPECT_FLOAT_EQ(0.5f, detections[1].score(0)); - EXPECT_FLOAT_EQ(1.0f, detections[2].score(0)); - EXPECT_EQ(1, detections[0].label_id(0)); - EXPECT_EQ(2, detections[1].label_id(0)); - EXPECT_EQ(3, detections[2].label_id(0)); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - OutputsDetectionsWithKeypointsAndCorrectValues) { - RunCalculatorWithKeypoints(); - const std::vector detections = runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>(); - EXPECT_EQ(kNumBoxes, detections.size()); - for (const auto& detection : detections) { - LocationData::RelativeBoundingBox relative_bbox = - detection.location_data().relative_bounding_box(); - EXPECT_FLOAT_EQ(0.2, relative_bbox.xmin()); - EXPECT_FLOAT_EQ(0.1, relative_bbox.ymin()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.width()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.height()); - for (const auto& relative_keypoint : - detection.location_data().relative_keypoints()) { - EXPECT_FLOAT_EQ(0.5, relative_keypoint.x()); - EXPECT_FLOAT_EQ(0.6, relative_keypoint.y()); - } - } - EXPECT_FLOAT_EQ(0.1f, detections[0].score(0)); - EXPECT_FLOAT_EQ(0.5f, detections[1].score(0)); - EXPECT_FLOAT_EQ(1.0f, detections[2].score(0)); - EXPECT_EQ(1, detections[0].label_id(0)); - EXPECT_EQ(2, detections[1].label_id(0)); - EXPECT_EQ(3, detections[2].label_id(0)); -} - -TEST_F(ObjectDetectionTensorsToDetectionsCalculatorTest, - SqueezesInputTensorDimensionAndOutputsDetectionsWithCorrectValues) { - RunCalculatorWithTensorDimensionSqueezing(); - const std::vector detections = runner_->Outputs() - .Tag(kDetections) - .packets[0] - .Get>(); - EXPECT_EQ(kNumBoxes, detections.size()); - for (const auto& detection : detections) { - LocationData::RelativeBoundingBox relative_bbox = - detection.location_data().relative_bounding_box(); - EXPECT_FLOAT_EQ(0.2, relative_bbox.xmin()); - EXPECT_FLOAT_EQ(0.1, relative_bbox.ymin()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.width()); - EXPECT_FLOAT_EQ(0.2, relative_bbox.height()); - } - EXPECT_FLOAT_EQ(0.1f, detections[0].score(0)); - EXPECT_FLOAT_EQ(0.5f, detections[1].score(0)); - EXPECT_FLOAT_EQ(1.0f, detections[2].score(0)); - EXPECT_EQ(1, detections[0].label_id(0)); - EXPECT_EQ(2, detections[1].label_id(0)); - EXPECT_EQ(3, detections[2].label_id(0)); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc deleted file mode 100644 index ddb042e6a..000000000 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ /dev/null @@ -1,520 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/container/flat_hash_map.h" -#include "absl/strings/match.h" -#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/sequence/media_sequence.h" -#include "mediapipe/util/sequence/media_sequence_util.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" - -namespace mediapipe { - -const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; -const char kImageTag[] = "IMAGE"; -const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_"; -const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_"; -const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED"; -const char kBBoxTag[] = "BBOX"; -const char kKeypointsTag[] = "KEYPOINTS"; -const char kSegmentationMaskTag[] = "CLASS_SEGMENTATION"; - -namespace tf = ::tensorflow; -namespace mpms = mediapipe::mediasequence; - -// Sink calculator to package streams into tf.SequenceExamples. -// -// The calculator takes a tf.SequenceExample as a side input and then adds -// the data from inputs to the SequenceExample with timestamps. Additional -// context features can be supplied verbatim in the calculator's options. The -// SequenceExample will conform to the description in media_sequence.h. -// -// The supported input stream tags are "IMAGE", which stores the encoded -// images from the OpenCVImageEncoderCalculator, "FORWARD_FLOW_ENCODED", which -// stores the encoded optical flow from the same calculator, "BBOX" which stores -// bounding boxes from vector, and streams with the -// "FLOAT_FEATURE_${NAME}" pattern, which stores the values from vector's -// associated with the name ${NAME}. "KEYPOINTS" stores a map of 2D keypoints -// from flat_hash_map>>. "IMAGE_${NAME}", -// "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store prefixed versions of -// each stream, which allows for multiple image streams to be included. However, -// the default names are suppored by more tools. -// -// Example config: -// node { -// calculator: "PackMediaSequenceCalculator" -// input_side_packet: "SEQUENCE_EXAMPLE:example_input_side_packet" -// input_stream: "IMAGE:frames" -// input_stream: "FLOAT_FEATURE_FDENSE:fdense_vf" -// output_stream: "SEQUENCE_EXAMPLE:example_output_stream" -// options { -// [mediapipe.PackMediaSequenceCalculatorOptions.ext]: { -// context_feature_map { -// feature { -// key: "image/frames_per_second" -// value { -// float_list { -// value: 30.0 -// } -// } -// } -// } -// } -// } -// } -namespace { -uint8 ConvertFloatToByte(const float float_value) { - float clamped_value = MathUtil::Clamp(0.0f, 1.0f, float_value); - return static_cast(clamped_value * 255.0 + .5f); -} -} // namespace - -class PackMediaSequenceCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->InputSidePackets().HasTag(kSequenceExampleTag)); - cc->InputSidePackets().Tag(kSequenceExampleTag).Set(); - - if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) { - cc->Inputs() - .Tag(kForwardFlowEncodedTag) - .Set(); - } - if (cc->Inputs().HasTag(kSegmentationMaskTag)) { - cc->Inputs().Tag(kSegmentationMaskTag).Set>(); - } - - for (const auto& tag : cc->Inputs().GetTags()) { - if (absl::StartsWith(tag, kImageTag)) { - std::string key = ""; - if (tag != kImageTag) { - int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kImageTag)_?" - } - } - cc->Inputs().Tag(tag).Set(); - } - if (absl::StartsWith(tag, kKeypointsTag)) { - std::string key = ""; - if (tag != kKeypointsTag) { - int tag_length = sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kKeypointsTag)_?" - } - } - cc->Inputs() - .Tag(tag) - .Set>>>(); - } - if (absl::StartsWith(tag, kBBoxTag)) { - std::string key = ""; - if (tag != kBBoxTag) { - int tag_length = sizeof(kBBoxTag) / sizeof(*kBBoxTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kBBoxTag)_?" - } - } - cc->Inputs().Tag(tag).Set>(); - } - if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) { - cc->Inputs().Tag(tag).Set>(); - } - if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) { - cc->Inputs().Tag(tag).Set>(); - } - } - - CHECK(cc->Outputs().HasTag(kSequenceExampleTag) || - cc->OutputSidePackets().HasTag(kSequenceExampleTag)) - << "Neither the output stream nor the output side packet is set to " - "output the sequence example."; - if (cc->Outputs().HasTag(kSequenceExampleTag)) { - cc->Outputs().Tag(kSequenceExampleTag).Set(); - } - if (cc->OutputSidePackets().HasTag(kSequenceExampleTag)) { - cc->OutputSidePackets() - .Tag(kSequenceExampleTag) - .Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - sequence_ = ::absl::make_unique( - cc->InputSidePackets() - .Tag(kSequenceExampleTag) - .Get()); - - const auto& context_features = - cc->Options().context_feature_map(); - for (const auto& feature : context_features.feature()) { - *mpms::MutableContext(feature.first, sequence_.get()) = feature.second; - } - for (const auto& tag : cc->Inputs().GetTags()) { - features_present_[tag] = false; - } - - replace_keypoints_ = false; - if (cc->Options() - .replace_data_instead_of_append()) { - for (const auto& tag : cc->Inputs().GetTags()) { - if (absl::StartsWith(tag, kImageTag)) { - std::string key = ""; - if (tag != kImageTag) { - int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kImageTag)_?" - } - } - mpms::ClearImageEncoded(key, sequence_.get()); - mpms::ClearImageTimestamp(key, sequence_.get()); - } - if (absl::StartsWith(tag, kBBoxTag)) { - std::string key = ""; - if (tag != kBBoxTag) { - int tag_length = sizeof(kBBoxTag) / sizeof(*kBBoxTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kBBoxTag)_?" - } - } - mpms::ClearBBox(key, sequence_.get()); - mpms::ClearBBoxTimestamp(key, sequence_.get()); - mpms::ClearBBoxIsAnnotated(key, sequence_.get()); - mpms::ClearBBoxNumRegions(key, sequence_.get()); - mpms::ClearBBoxLabelString(key, sequence_.get()); - mpms::ClearBBoxLabelIndex(key, sequence_.get()); - mpms::ClearBBoxClassString(key, sequence_.get()); - mpms::ClearBBoxClassIndex(key, sequence_.get()); - mpms::ClearBBoxTrackString(key, sequence_.get()); - mpms::ClearBBoxTrackIndex(key, sequence_.get()); - mpms::ClearUnmodifiedBBoxTimestamp(key, sequence_.get()); - } - if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) { - std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) / - sizeof(*kFloatFeaturePrefixTag) - - 1); - mpms::ClearFeatureFloats(key, sequence_.get()); - mpms::ClearFeatureTimestamp(key, sequence_.get()); - } - if (absl::StartsWith(tag, kKeypointsTag)) { - std::string key = - tag.substr(sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1); - replace_keypoints_ = true; - } - } - if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) { - mpms::ClearForwardFlowEncoded(sequence_.get()); - mpms::ClearForwardFlowTimestamp(sequence_.get()); - } - } - - if (cc->Outputs().HasTag(kSequenceExampleTag)) { - cc->Outputs() - .Tag(kSequenceExampleTag) - .SetNextTimestampBound(Timestamp::Max()); - } - return absl::OkStatus(); - } - - absl::Status VerifySequence() { - std::string error_msg = "Missing features - "; - bool all_present = true; - for (const auto& iter : features_present_) { - if (!iter.second) { - all_present = false; - absl::StrAppend(&error_msg, iter.first, ", "); - } - } - if (all_present) { - return absl::OkStatus(); - } else { - return ::mediapipe::NotFoundErrorBuilder(MEDIAPIPE_LOC) << error_msg; - } - } - - absl::Status VerifySize() { - const int64 MAX_PROTO_BYTES = 1073741823; - std::string id = mpms::HasExampleId(*sequence_) - ? mpms::GetExampleId(*sequence_) - : "example"; - RET_CHECK_LT(sequence_->ByteSizeLong(), MAX_PROTO_BYTES) - << "sequence '" << id - << "' would be too many bytes to serialize after adding features."; - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) override { - auto& options = cc->Options(); - if (options.reconcile_metadata()) { - RET_CHECK_OK(mpms::ReconcileMetadata( - options.reconcile_bbox_annotations(), - options.reconcile_region_annotations(), sequence_.get())); - } - - if (options.skip_large_sequences()) { - RET_CHECK_OK(VerifySize()); - } - if (options.output_only_if_all_present()) { - absl::Status status = VerifySequence(); - if (!status.ok()) { - cc->GetCounter(status.ToString())->Increment(); - return status; - } - } - - if (cc->OutputSidePackets().HasTag(kSequenceExampleTag)) { - cc->OutputSidePackets() - .Tag(kSequenceExampleTag) - .Set(MakePacket(*sequence_)); - } - if (cc->Outputs().HasTag(kSequenceExampleTag)) { - cc->Outputs() - .Tag(kSequenceExampleTag) - .Add(sequence_.release(), Timestamp::PostStream()); - } - sequence_.reset(); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - int image_height = -1; - int image_width = -1; - // Because the tag order may vary, we need to loop through tags to get - // image information before processing other tag types. - for (const auto& tag : cc->Inputs().GetTags()) { - if (!cc->Inputs().Tag(tag).IsEmpty()) { - features_present_[tag] = true; - } - if (absl::StartsWith(tag, kImageTag) && - !cc->Inputs().Tag(tag).IsEmpty()) { - std::string key = ""; - if (tag != kImageTag) { - int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kImageTag)_?" - } - } - const OpenCvImageEncoderCalculatorResults& image = - cc->Inputs().Tag(tag).Get(); - if (!image.has_encoded_image()) { - return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "No encoded image"; - } - image_height = image.height(); - image_width = image.width(); - mpms::AddImageTimestamp(key, cc->InputTimestamp().Value(), - sequence_.get()); - mpms::AddImageEncoded(key, image.encoded_image(), sequence_.get()); - } - } - for (const auto& tag : cc->Inputs().GetTags()) { - if (!cc->Inputs().Tag(tag).IsEmpty()) { - features_present_[tag] = true; - } - if (absl::StartsWith(tag, kKeypointsTag) && - !cc->Inputs().Tag(tag).IsEmpty()) { - std::string key = ""; - if (tag != kKeypointsTag) { - int tag_length = sizeof(kKeypointsTag) / sizeof(*kKeypointsTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kKeypointsTag)_?" - } - } - const auto& keypoints = - cc->Inputs() - .Tag(tag) - .Get>>>(); - for (const auto& pair : keypoints) { - std::string prefix = mpms::merge_prefix(key, pair.first); - if (replace_keypoints_) { - mpms::ClearBBoxPoint(prefix, sequence_.get()); - mpms::ClearBBoxTimestamp(prefix, sequence_.get()); - mpms::ClearBBoxIsAnnotated(prefix, sequence_.get()); - mpms::ClearBBoxNumRegions(prefix, sequence_.get()); - mpms::ClearBBoxLabelString(prefix, sequence_.get()); - mpms::ClearBBoxLabelIndex(prefix, sequence_.get()); - mpms::ClearBBoxClassString(prefix, sequence_.get()); - mpms::ClearBBoxClassIndex(prefix, sequence_.get()); - mpms::ClearBBoxTrackString(prefix, sequence_.get()); - mpms::ClearBBoxTrackIndex(prefix, sequence_.get()); - mpms::ClearUnmodifiedBBoxTimestamp(prefix, sequence_.get()); - } - mpms::AddBBoxTimestamp(prefix, cc->InputTimestamp().Value(), - sequence_.get()); - mpms::AddBBoxPoint(prefix, pair.second, sequence_.get()); - } - replace_keypoints_ = false; - } - if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) && - !cc->Inputs().Tag(tag).IsEmpty()) { - std::string key = - tag.substr(sizeof(kFloatContextFeaturePrefixTag) / - sizeof(*kFloatContextFeaturePrefixTag) - - 1); - RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream()); - mpms::SetContextFeatureFloats( - key, cc->Inputs().Tag(tag).Get>(), - sequence_.get()); - } - if (absl::StartsWith(tag, kFloatFeaturePrefixTag) && - !cc->Inputs().Tag(tag).IsEmpty()) { - std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) / - sizeof(*kFloatFeaturePrefixTag) - - 1); - mpms::AddFeatureTimestamp(key, cc->InputTimestamp().Value(), - sequence_.get()); - mpms::AddFeatureFloats(key, - cc->Inputs().Tag(tag).Get>(), - sequence_.get()); - } - if (absl::StartsWith(tag, kBBoxTag) && !cc->Inputs().Tag(tag).IsEmpty()) { - std::string key = ""; - if (tag != kBBoxTag) { - int tag_length = sizeof(kBBoxTag) / sizeof(*kBBoxTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kBBoxTag)_?" - } - } - std::vector predicted_locations; - std::vector predicted_class_strings; - std::vector predicted_label_ids; - for (auto& detection : - cc->Inputs().Tag(tag).Get>()) { - if (detection.location_data().format() == - LocationData::BOUNDING_BOX || - detection.location_data().format() == - LocationData::RELATIVE_BOUNDING_BOX) { - if (mpms::HasImageHeight(*sequence_) && - mpms::HasImageWidth(*sequence_)) { - image_height = mpms::GetImageHeight(*sequence_); - image_width = mpms::GetImageWidth(*sequence_); - } - if (image_height == -1 || image_width == -1) { - return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Images must be provided with bounding boxes or the " - "image " - << "height and width must already be in the example."; - } - Location relative_bbox = Location::CreateRelativeBBoxLocation( - Location(detection.location_data()) - .ConvertToRelativeBBox(image_width, image_height)); - predicted_locations.push_back(relative_bbox); - if (detection.label_size() > 0) { - predicted_class_strings.push_back(detection.label(0)); - } - if (detection.label_id_size() > 0) { - predicted_label_ids.push_back(detection.label_id(0)); - } - } - } - if (!predicted_locations.empty()) { - mpms::AddBBox(key, predicted_locations, sequence_.get()); - mpms::AddBBoxTimestamp(key, cc->InputTimestamp().Value(), - sequence_.get()); - if (!predicted_class_strings.empty()) { - mpms::AddBBoxLabelString(key, predicted_class_strings, - sequence_.get()); - } - if (!predicted_label_ids.empty()) { - mpms::AddBBoxLabelIndex(key, predicted_label_ids, sequence_.get()); - } - } - } - } - if (cc->Inputs().HasTag(kForwardFlowEncodedTag) && - !cc->Inputs().Tag(kForwardFlowEncodedTag).IsEmpty()) { - const OpenCvImageEncoderCalculatorResults& forward_flow = - cc->Inputs() - .Tag(kForwardFlowEncodedTag) - .Get(); - if (!forward_flow.has_encoded_image()) { - return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "No encoded forward flow"; - } - mpms::AddForwardFlowTimestamp(cc->InputTimestamp().Value(), - sequence_.get()); - mpms::AddForwardFlowEncoded(forward_flow.encoded_image(), - sequence_.get()); - } - if (cc->Inputs().HasTag(kSegmentationMaskTag) && - !cc->Inputs().Tag(kSegmentationMaskTag).IsEmpty()) { - bool already_has_mask = false; - for (auto& detection : cc->Inputs() - .Tag(kSegmentationMaskTag) - .Get>()) { - if (detection.location_data().format() == LocationData::MASK) { - RET_CHECK(!already_has_mask) - << "We currently only support adding one mask per timestamp. " - << sequence_->DebugString(); - auto mask_mat_ptr = Location(detection.location_data()).GetCvMask(); - std::vector bytes; - RET_CHECK(cv::imencode(".png", *mask_mat_ptr, bytes, {})); - - std::string encoded_mask(bytes.begin(), bytes.end()); - mpms::AddClassSegmentationEncoded(encoded_mask, sequence_.get()); - mpms::AddClassSegmentationTimestamp(cc->InputTimestamp().Value(), - sequence_.get()); - // SegmentationClassLabelString is a context feature for the entire - // sequence. The values in the last detection will be saved. - mpms::SetClassSegmentationClassLabelString({detection.label(0)}, - sequence_.get()); - already_has_mask = true; - } else { - return absl::UnimplementedError( - "Global detections and empty detections are not supported."); - } - } - } - return absl::OkStatus(); - } - - std::unique_ptr sequence_; - std::map features_present_; - bool replace_keypoints_; -}; -REGISTER_CALCULATOR(PackMediaSequenceCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto deleted file mode 100644 index 695eb6b5e..000000000 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.proto +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "tensorflow/core/example/feature.proto"; - -message PackMediaSequenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional PackMediaSequenceCalculatorOptions ext = 243132285; - } - - // Provide a map of strings to tf.Features to merge into the SequenceExample's - // context. Use this to add new metadata. - optional tensorflow.Features context_feature_map = 1; - - // If true, update the context for the SequenceExample features. - // (e.g. fills in the image height, width, and number of frames.) - optional bool reconcile_metadata = 2 [default = true]; - - // If true, updates the metadata for bounding box portions of sequences. This - // will align each bounding box annotation with the nearest frame and insert - // empty annotations as needed to satisfy the frame rate. - // NOTE: IF YOU DOWNSAMPLE IN TIME YOU WILL LOSE ANNOTATIONS. - // If two or more annotations are closest to the same frame, then only - // the closest annotation is saved. This matches the behavior of - // downsampling images in time. - optional bool reconcile_bbox_annotations = 5 [default = false]; - - // If true, updates the metadata for all regions annotations, regardless of - // prefix, in the sequence. This will align each region annotation with the - // nearest frame and insert empty annotations as needed to satisfy the frame - // rate. This is particularly useful for key point annotations that are - // represented as region points. This does not exclude bboxes. - // NOTE: IF YOU DOWNSAMPLE IN TIME YOU WILL LOSE ANNOTATIONS. - // If two or more annotations are closest to the same frame, then only - // the closest annotation is saved. This matches the behavior of - // downsampling images in time. - optional bool reconcile_region_annotations = 6 [default = true]; - - // If true, the SequenceExample is output only if all input streams are - // present. - optional bool output_only_if_all_present = 3 [default = true]; - - // If true, will remove all data from a sequence example for a corresponding - // input stream. E.g. if images are already present and an IMAGE stream is - // present, the previous images and timestamps will be removed before adding - // the new images. - optional bool replace_data_instead_of_append = 4 [default = true]; - - // If true, will return an error status if an output sequence would be too - // many bytes to serialize. - optional bool skip_large_sequences = 7 [default = true]; -} diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc deleted file mode 100644 index c163cebcd..000000000 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc +++ /dev/null @@ -1,912 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/container/flat_hash_map.h" -#include "absl/memory/memory.h" -#include "absl/strings/numbers.h" -#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/sequence/media_sequence.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" - -namespace mediapipe { -namespace { - -namespace tf = ::tensorflow; -namespace mpms = mediapipe::mediasequence; - -class PackMediaSequenceCalculatorTest : public ::testing::Test { - protected: - void SetUpCalculator(const std::vector& input_streams, - const tf::Features& features, - bool output_only_if_all_present, - bool replace_instead_of_append) { - CalculatorGraphConfig::Node config; - config.set_calculator("PackMediaSequenceCalculator"); - config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence"); - config.add_output_stream("SEQUENCE_EXAMPLE:output_sequence"); - for (const std::string& stream : input_streams) { - config.add_input_stream(stream); - } - auto options = config.mutable_options()->MutableExtension( - PackMediaSequenceCalculatorOptions::ext); - *options->mutable_context_feature_map() = features; - options->set_output_only_if_all_present(output_only_if_all_present); - options->set_replace_data_instead_of_append(replace_instead_of_append); - runner_ = ::absl::make_unique(config); - } - - std::unique_ptr runner_; -}; - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) { - SetUpCalculator({"IMAGE:images"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - encoded_image.set_width(2); - encoded_image.set_height(1); - - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs()->Tag("IMAGE").packets.push_back( - Adopt(image_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(num_images, mpms::GetImageTimestampSize(output_sequence)); - ASSERT_EQ(num_images, mpms::GetImageEncodedSize(output_sequence)); - for (int i = 0; i < num_images; ++i) { - ASSERT_EQ(i, mpms::GetImageTimestampAt(output_sequence, i)); - ASSERT_EQ(encoded_image.encoded_image(), - mpms::GetImageEncodedAt(output_sequence, i)); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) { - std::string prefix = "PREFIX"; - SetUpCalculator({"IMAGE_PREFIX:images"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - encoded_image.set_width(2); - encoded_image.set_height(1); - - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs() - ->Tag("IMAGE_PREFIX") - .packets.push_back(Adopt(image_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(num_images, mpms::GetImageTimestampSize(prefix, output_sequence)); - ASSERT_EQ(num_images, mpms::GetImageEncodedSize(prefix, output_sequence)); - for (int i = 0; i < num_images; ++i) { - ASSERT_EQ(i, mpms::GetImageTimestampAt(prefix, output_sequence, i)); - ASSERT_EQ(encoded_image.encoded_image(), - mpms::GetImageEncodedAt(prefix, output_sequence, i)); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) { - SetUpCalculator({"FLOAT_FEATURE_TEST:test", "FLOAT_FEATURE_OTHER:test2"}, {}, - false, true); - auto input_sequence = ::absl::make_unique(); - - int num_timesteps = 2; - for (int i = 0; i < num_timesteps; ++i) { - auto vf_ptr = ::absl::make_unique>(2, 2 << i); - runner_->MutableInputs() - ->Tag("FLOAT_FEATURE_TEST") - .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); - vf_ptr = ::absl::make_unique>(2, 2 << i); - runner_->MutableInputs() - ->Tag("FLOAT_FEATURE_OTHER") - .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(num_timesteps, - mpms::GetFeatureTimestampSize("TEST", output_sequence)); - ASSERT_EQ(num_timesteps, mpms::GetFeatureFloatsSize("TEST", output_sequence)); - ASSERT_EQ(num_timesteps, - mpms::GetFeatureTimestampSize("OTHER", output_sequence)); - ASSERT_EQ(num_timesteps, - mpms::GetFeatureFloatsSize("OTHER", output_sequence)); - for (int i = 0; i < num_timesteps; ++i) { - ASSERT_EQ(i, mpms::GetFeatureTimestampAt("TEST", output_sequence, i)); - ASSERT_THAT(mpms::GetFeatureFloatsAt("TEST", output_sequence, i), - ::testing::ElementsAreArray(std::vector(2, 2 << i))); - ASSERT_EQ(i, mpms::GetFeatureTimestampAt("OTHER", output_sequence, i)); - ASSERT_THAT(mpms::GetFeatureFloatsAt("OTHER", output_sequence, i), - ::testing::ElementsAreArray(std::vector(2, 2 << i))); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) { - SetUpCalculator( - {"FLOAT_CONTEXT_FEATURE_TEST:test", "FLOAT_CONTEXT_FEATURE_OTHER:test2"}, - {}, false, true); - auto input_sequence = absl::make_unique(); - - auto vf_ptr = absl::make_unique>(2, 3); - runner_->MutableInputs() - ->Tag("FLOAT_CONTEXT_FEATURE_TEST") - .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream())); - vf_ptr = absl::make_unique>(2, 4); - runner_->MutableInputs() - ->Tag("FLOAT_CONTEXT_FEATURE_OTHER") - .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp::PostStream())); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence), - testing::ElementsAre(3, 3)); - ASSERT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence), - testing::ElementsAre(4, 4)); -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) { - tf::Features context; - (*context.mutable_feature())["TEST"].mutable_bytes_list()->add_value("YES"); - (*context.mutable_feature())["OTHER"].mutable_bytes_list()->add_value("NO"); - SetUpCalculator({"IMAGE:images"}, context, false, true); - - auto input_sequence = ::absl::make_unique(); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs()->Tag("IMAGE").packets.push_back( - Adopt(image_ptr.release()).At(Timestamp(0))); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_TRUE(mpms::HasContext(output_sequence, "TEST")); - ASSERT_TRUE(mpms::HasContext(output_sequence, "OTHER")); - ASSERT_EQ(mpms::GetContext(output_sequence, "TEST").bytes_list().value(0), - "YES"); - ASSERT_EQ(mpms::GetContext(output_sequence, "OTHER").bytes_list().value(0), - "NO"); -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) { - SetUpCalculator({"FORWARD_FLOW_ENCODED:flow"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - std::string test_flow_string(bytes.begin(), bytes.end()); - OpenCvImageEncoderCalculatorResults encoded_flow; - encoded_flow.set_encoded_image(test_flow_string); - encoded_flow.set_width(2); - encoded_flow.set_height(1); - - int num_flows = 2; - for (int i = 0; i < num_flows; ++i) { - auto flow_ptr = - ::absl::make_unique(encoded_flow); - runner_->MutableInputs() - ->Tag("FORWARD_FLOW_ENCODED") - .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(num_flows, mpms::GetForwardFlowTimestampSize(output_sequence)); - ASSERT_EQ(num_flows, mpms::GetForwardFlowEncodedSize(output_sequence)); - for (int i = 0; i < num_flows; ++i) { - ASSERT_EQ(i, mpms::GetForwardFlowTimestampAt(output_sequence, i)); - ASSERT_EQ(test_flow_string, - mpms::GetForwardFlowEncodedAt(output_sequence, i)); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) { - SetUpCalculator({"BBOX_PREDICTED:detections"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - int height = 480; - int width = 640; - mpms::SetImageHeight(height, input_sequence.get()); - mpms::SetImageWidth(width, input_sequence.get()); - - int num_vectors = 2; - for (int i = 0; i < num_vectors; ++i) { - auto detections = ::absl::make_unique<::std::vector>(); - Detection detection; - detection.add_label("absolute bbox"); - detection.add_label_id(0); - detection.add_score(0.5); - Location::CreateBBoxLocation(0, height / 2, width / 2, height / 2) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - detection = Detection(); - detection.add_label("relative bbox"); - detection.add_label_id(1); - detection.add_score(0.75); - Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - // The mask detection should be ignored in the output. - detection = Detection(); - detection.add_label("mask"); - detection.add_score(1.0); - cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0)); - Location::CreateCvMaskLocation(image).ConvertToProto( - detection.mutable_location_data()); - detections->push_back(detection); - - runner_->MutableInputs() - ->Tag("BBOX_PREDICTED") - .packets.push_back(Adopt(detections.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(height, mpms::GetImageHeight(output_sequence)); - ASSERT_EQ(width, mpms::GetImageWidth(output_sequence)); - ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxSize(output_sequence)); - ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxTimestampSize(output_sequence)); - ASSERT_EQ(0, mpms::GetClassSegmentationEncodedSize(output_sequence)); - ASSERT_EQ(0, mpms::GetClassSegmentationTimestampSize(output_sequence)); - for (int i = 0; i < num_vectors; ++i) { - ASSERT_EQ(i, mpms::GetPredictedBBoxTimestampAt(output_sequence, i)); - auto bboxes = mpms::GetPredictedBBoxAt(output_sequence, i); - ASSERT_EQ(2, bboxes.size()); - for (int j = 0; j < bboxes.size(); ++j) { - auto rect = bboxes[j].GetRelativeBBox(); - ASSERT_NEAR(0, rect.xmin(), 0.001); - ASSERT_NEAR(0.5, rect.ymin(), 0.001); - ASSERT_NEAR(0.5, rect.xmax(), 0.001); - ASSERT_NEAR(1.0, rect.ymax(), 0.001); - } - auto class_strings = - mpms::GetPredictedBBoxLabelStringAt(output_sequence, i); - ASSERT_EQ("absolute bbox", class_strings[0]); - ASSERT_EQ("relative bbox", class_strings[1]); - auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); - ASSERT_EQ(0, class_indices[0]); - ASSERT_EQ(1, class_indices[1]); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) { - SetUpCalculator({"BBOX_PREDICTED:detections"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - int height = 480; - int width = 640; - int num_vectors = 2; - for (int i = 0; i < num_vectors; ++i) { - auto detections = ::absl::make_unique<::std::vector>(); - Detection detection; - detection.add_label("absolute bbox"); - detection.add_label_id(0); - detection.add_score(0.5); - Location::CreateBBoxLocation(0, height / 2, width / 2, height / 2) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - detection = Detection(); - detection.add_label("relative bbox"); - detection.add_label_id(1); - detection.add_score(0.75); - Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - // The mask detection should be ignored in the output. - detection = Detection(); - detection.add_label("mask"); - detection.add_score(1.0); - cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0)); - Location::CreateCvMaskLocation(image).ConvertToProto( - detection.mutable_location_data()); - detections->push_back(detection); - - runner_->MutableInputs() - ->Tag("BBOX_PREDICTED") - .packets.push_back(Adopt(detections.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - auto status = runner_->Run(); - EXPECT_EQ(absl::StatusCode::kInvalidArgument, status.code()); -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) { - SetUpCalculator({"BBOX_PREDICTED:detections", "IMAGE:images"}, {}, false, - true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - int height = 480; - int width = 640; - int num_vectors = 2; - for (int i = 0; i < num_vectors; ++i) { - auto detections = ::absl::make_unique<::std::vector>(); - Detection detection; - detection.add_label("absolute bbox"); - detection.add_label_id(0); - detection.add_score(0.5); - Location::CreateBBoxLocation(0, height / 2, width / 2, height / 2) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - detection = Detection(); - detection.add_label("relative bbox"); - detection.add_label_id(1); - detection.add_score(0.75); - Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - - // The mask detection should be ignored in the output. - detection = Detection(); - detection.add_label("mask"); - detection.add_score(1.0); - cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0)); - Location::CreateCvMaskLocation(image).ConvertToProto( - detection.mutable_location_data()); - detections->push_back(detection); - - runner_->MutableInputs() - ->Tag("BBOX_PREDICTED") - .packets.push_back(Adopt(detections.release()).At(Timestamp(i))); - } - cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - encoded_image.set_width(width); - encoded_image.set_height(height); - - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs()->Tag("IMAGE").packets.push_back( - Adopt(image_ptr.release()).At(Timestamp(i))); - } - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(height, mpms::GetImageHeight(output_sequence)); - ASSERT_EQ(width, mpms::GetImageWidth(output_sequence)); - ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxSize(output_sequence)); - ASSERT_EQ(num_vectors, mpms::GetPredictedBBoxTimestampSize(output_sequence)); - ASSERT_EQ(0, mpms::GetClassSegmentationEncodedSize(output_sequence)); - ASSERT_EQ(0, mpms::GetClassSegmentationTimestampSize(output_sequence)); - for (int i = 0; i < num_vectors; ++i) { - ASSERT_EQ(i, mpms::GetPredictedBBoxTimestampAt(output_sequence, i)); - auto bboxes = mpms::GetPredictedBBoxAt(output_sequence, i); - ASSERT_EQ(2, bboxes.size()); - for (int j = 0; j < bboxes.size(); ++j) { - auto rect = bboxes[j].GetRelativeBBox(); - ASSERT_NEAR(0, rect.xmin(), 0.001); - ASSERT_NEAR(0.5, rect.ymin(), 0.001); - ASSERT_NEAR(0.5, rect.xmax(), 0.001); - ASSERT_NEAR(1.0, rect.ymax(), 0.001); - } - auto class_strings = - mpms::GetPredictedBBoxLabelStringAt(output_sequence, i); - ASSERT_EQ("absolute bbox", class_strings[0]); - ASSERT_EQ("relative bbox", class_strings[1]); - auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); - ASSERT_EQ(0, class_indices[0]); - ASSERT_EQ(1, class_indices[1]); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoKeypoints) { - SetUpCalculator({"KEYPOINTS_TEST:keypoints"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - absl::flat_hash_map>> - points = {{"HEAD", {{0.1, 0.2}, {0.3, 0.4}}}, {"TAIL", {{0.5, 0.6}}}}; - runner_->MutableInputs() - ->Tag("KEYPOINTS_TEST") - .packets.push_back(PointToForeign(&points).At(Timestamp(0))); - runner_->MutableInputs() - ->Tag("KEYPOINTS_TEST") - .packets.push_back(PointToForeign(&points).At(Timestamp(1))); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(2, mpms::GetBBoxPointSize("TEST/HEAD", output_sequence)); - ASSERT_EQ(2, mpms::GetBBoxPointSize("TEST/TAIL", output_sequence)); - ASSERT_NEAR(0.2, - mpms::GetBBoxPointAt("TEST/HEAD", output_sequence, 0)[0].second, - 0.001); - ASSERT_NEAR(0.5, - mpms::GetBBoxPointAt("TEST/TAIL", output_sequence, 1)[0].first, - 0.001); -} - -TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) { - SetUpCalculator({"CLASS_SEGMENTATION:detections"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - int height = 480; - int width = 640; - mpms::SetImageHeight(height, input_sequence.get()); - mpms::SetImageWidth(width, input_sequence.get()); - - int num_vectors = 2; - for (int i = 0; i < num_vectors; ++i) { - auto detections = ::absl::make_unique<::std::vector>(); - Detection detection; - detection = Detection(); - detection.add_label("mask"); - detection.add_score(1.0); - cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0)); - Location::CreateCvMaskLocation(image).ConvertToProto( - detection.mutable_location_data()); - - detections->push_back(detection); - - runner_->MutableInputs() - ->Tag("CLASS_SEGMENTATION") - .packets.push_back(Adopt(detections.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(height, mpms::GetImageHeight(output_sequence)); - ASSERT_EQ(width, mpms::GetImageWidth(output_sequence)); - ASSERT_EQ(2, mpms::GetClassSegmentationEncodedSize(output_sequence)); - ASSERT_EQ(2, mpms::GetClassSegmentationTimestampSize(output_sequence)); - for (int i = 0; i < num_vectors; ++i) { - ASSERT_EQ(i, mpms::GetClassSegmentationTimestampAt(output_sequence, i)); - } - ASSERT_THAT(mpms::GetClassSegmentationClassLabelString(output_sequence), - testing::ElementsAreArray(::std::vector({"mask"}))); -} - -TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) { - SetUpCalculator( - {"FORWARD_FLOW_ENCODED:flow", "FLOAT_FEATURE_I3D_FLOW:feature"}, {}, - false, false); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - std::string test_flow_string(bytes.begin(), bytes.end()); - OpenCvImageEncoderCalculatorResults encoded_flow; - encoded_flow.set_encoded_image(test_flow_string); - encoded_flow.set_width(2); - encoded_flow.set_height(1); - - int num_flows = 2; - for (int i = 0; i < num_flows; ++i) { - auto flow_ptr = - ::absl::make_unique(encoded_flow); - runner_->MutableInputs() - ->Tag("FORWARD_FLOW_ENCODED") - .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(num_flows, mpms::GetForwardFlowTimestampSize(output_sequence)); - ASSERT_EQ(num_flows, mpms::GetForwardFlowEncodedSize(output_sequence)); - for (int i = 0; i < num_flows; ++i) { - ASSERT_EQ(i, mpms::GetForwardFlowTimestampAt(output_sequence, i)); - ASSERT_EQ(test_flow_string, - mpms::GetForwardFlowEncodedAt(output_sequence, i)); - } -} - -TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) { - SetUpCalculator( - {"FORWARD_FLOW_ENCODED:flow", "FLOAT_FEATURE_I3D_FLOW:feature"}, {}, true, - false); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - std::string test_flow_string(bytes.begin(), bytes.end()); - OpenCvImageEncoderCalculatorResults encoded_flow; - encoded_flow.set_encoded_image(test_flow_string); - encoded_flow.set_width(2); - encoded_flow.set_height(1); - - int num_flows = 2; - for (int i = 0; i < num_flows; ++i) { - auto flow_ptr = - ::absl::make_unique(encoded_flow); - runner_->MutableInputs() - ->Tag("FORWARD_FLOW_ENCODED") - .packets.push_back(Adopt(flow_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - absl::Status status = runner_->Run(); - EXPECT_FALSE(status.ok()); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestReplacingImages) { - SetUpCalculator({"IMAGE:images"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - mpms::AddImageEncoded("one", input_sequence.get()); - mpms::AddImageEncoded("two", input_sequence.get()); - mpms::AddImageTimestamp(1, input_sequence.get()); - mpms::AddImageTimestamp(2, input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(0, mpms::GetImageTimestampSize(output_sequence)); - ASSERT_EQ(0, mpms::GetImageEncodedSize(output_sequence)); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFlowImages) { - SetUpCalculator({"FORWARD_FLOW_ENCODED:images"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - mpms::AddForwardFlowEncoded("one", input_sequence.get()); - mpms::AddForwardFlowEncoded("two", input_sequence.get()); - mpms::AddForwardFlowTimestamp(1, input_sequence.get()); - mpms::AddForwardFlowTimestamp(2, input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); - ASSERT_EQ(0, mpms::GetForwardFlowTimestampSize(output_sequence)); - ASSERT_EQ(0, mpms::GetForwardFlowEncodedSize(output_sequence)); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestReplacingFloatVectors) { - SetUpCalculator({"FLOAT_FEATURE_TEST:test", "FLOAT_FEATURE_OTHER:test2"}, {}, - false, true); - auto input_sequence = ::absl::make_unique(); - - int num_timesteps = 2; - for (int i = 0; i < num_timesteps; ++i) { - auto vf_ptr = ::absl::make_unique>(2, 2 << i); - mpms::AddFeatureFloats("TEST", *vf_ptr, input_sequence.get()); - mpms::AddFeatureTimestamp("TEST", i, input_sequence.get()); - vf_ptr = ::absl::make_unique>(2, 2 << i); - mpms::AddFeatureFloats("OTHER", *vf_ptr, input_sequence.get()); - mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get()); - } - ASSERT_EQ(num_timesteps, - mpms::GetFeatureTimestampSize("TEST", *input_sequence)); - ASSERT_EQ(num_timesteps, mpms::GetFeatureFloatsSize("TEST", *input_sequence)); - ASSERT_EQ(num_timesteps, - mpms::GetFeatureTimestampSize("OTHER", *input_sequence)); - ASSERT_EQ(num_timesteps, - mpms::GetFeatureFloatsSize("OTHER", *input_sequence)); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(0, mpms::GetFeatureTimestampSize("TEST", output_sequence)); - ASSERT_EQ(0, mpms::GetFeatureFloatsSize("TEST", output_sequence)); - ASSERT_EQ(0, mpms::GetFeatureTimestampSize("OTHER", output_sequence)); - ASSERT_EQ(0, mpms::GetFeatureFloatsSize("OTHER", output_sequence)); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) { - SetUpCalculator({"IMAGE:images"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - encoded_image.set_width(2); - encoded_image.set_height(1); - - int num_images = 5; // Timestamps: 10, 20, 30, 40, 50 - for (int i = 0; i < num_images; ++i) { - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs()->Tag("IMAGE").packets.push_back( - Adopt(image_ptr.release()).At(Timestamp((i + 1) * 10))); - } - - mpms::AddBBoxTimestamp(9, input_sequence.get()); - mpms::AddBBoxTimestamp(21, input_sequence.get()); - mpms::AddBBoxTimestamp(22, input_sequence.get()); - - mpms::AddBBoxTimestamp("PREFIX", 8, input_sequence.get()); - mpms::AddBBoxTimestamp("PREFIX", 9, input_sequence.get()); - mpms::AddBBoxTimestamp("PREFIX", 22, input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - MP_ASSERT_OK(runner_->Run()); - const std::vector& output_packets = - runner_->Outputs().Tag("SEQUENCE_EXAMPLE").packets; - ASSERT_EQ(1, output_packets.size()); - const tf::SequenceExample& output_sequence = - output_packets[0].Get(); - - ASSERT_EQ(mpms::GetBBoxTimestampSize(output_sequence), 5); - ASSERT_EQ(mpms::GetBBoxTimestampAt(output_sequence, 0), 10); - ASSERT_EQ(mpms::GetBBoxTimestampAt(output_sequence, 1), 20); - ASSERT_EQ(mpms::GetBBoxTimestampAt(output_sequence, 2), 30); - ASSERT_EQ(mpms::GetBBoxTimestampAt(output_sequence, 3), 40); - ASSERT_EQ(mpms::GetBBoxTimestampAt(output_sequence, 4), 50); - - ASSERT_EQ(mpms::GetBBoxTimestampSize("PREFIX", output_sequence), 5); - ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 0), 10); - ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 1), 20); - ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 2), 30); - ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 3), 40); - ASSERT_EQ(mpms::GetBBoxTimestampAt("PREFIX", output_sequence, 4), 50); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) { - SetUpCalculator({"IMAGE:images", "BBOX:bbox"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); - std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); - OpenCvImageEncoderCalculatorResults encoded_image; - encoded_image.set_encoded_image(bytes.data(), bytes.size()); - int height = 2; - int width = 2; - encoded_image.set_width(width); - encoded_image.set_height(height); - - int num_images = 5; // Timestamps: 10, 20, 30, 40, 50 - for (int i = 0; i < num_images; ++i) { - auto image_ptr = - ::absl::make_unique(encoded_image); - runner_->MutableInputs()->Tag("IMAGE").packets.push_back( - Adopt(image_ptr.release()).At(Timestamp(i))); - } - - for (int i = 0; i < num_images; ++i) { - auto detections = ::absl::make_unique<::std::vector>(); - Detection detection; - detection = Detection(); - detection.add_label("relative bbox"); - detection.add_label_id(1); - detection.add_score(0.75); - Location::CreateRelativeBBoxLocation(0, 0.5, 0.5, 0.5) - .ConvertToProto(detection.mutable_location_data()); - detections->push_back(detection); - runner_->MutableInputs()->Tag("BBOX").packets.push_back( - Adopt(detections.release()).At(Timestamp(i))); - } - - for (int i = 0; i < 10; ++i) { - mpms::AddBBoxTimestamp(-1, input_sequence.get()); - mpms::AddBBoxIsAnnotated(-1, input_sequence.get()); - mpms::AddBBoxNumRegions(-1, input_sequence.get()); - mpms::AddBBoxLabelString({"anything"}, input_sequence.get()); - mpms::AddBBoxLabelIndex({-1}, input_sequence.get()); - mpms::AddBBoxClassString({"anything"}, input_sequence.get()); - mpms::AddBBoxClassIndex({-1}, input_sequence.get()); - mpms::AddBBoxTrackString({"anything"}, input_sequence.get()); - mpms::AddBBoxTrackIndex({-1}, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - // If the all the previous values aren't cleared, this assert will fail. - MP_ASSERT_OK(runner_->Run()); -} - -TEST_F(PackMediaSequenceCalculatorTest, TestTooLargeInputFailsSoftly) { - SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true); - auto input_sequence = ::absl::make_unique(); - - // 1 billion floats should be > 1GB which can't be serialized. It should fail - // gracefully with this input. - int num_timesteps = 1000; - for (int i = 0; i < num_timesteps; ++i) { - auto vf_ptr = ::absl::make_unique>(1000000, i); - runner_->MutableInputs() - ->Tag("FLOAT_FEATURE_TEST") - .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - ASSERT_FALSE(runner_->Run().ok()); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/string_to_sequence_example_calculator.cc b/mediapipe/calculators/tensorflow/string_to_sequence_example_calculator.cc deleted file mode 100644 index da85bed94..000000000 --- a/mediapipe/calculators/tensorflow/string_to_sequence_example_calculator.cc +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "tensorflow/core/example/example.pb.h" - -// A calculator to serialize/deserialize tensorflow::SequenceExample protos -// to and from strings. -// -// Example converting to SequenceExample in Open(): -// node { -// calculator: "StringToSequenceExampleCalculator" -// input_side_packet: "STRING:serialized_sequence_example" -// output_side_packet: "SEQUENCE_EXAMPLE:sequence_example" -// } -// -// Example converting to std::string in Close(): -// node { -// calculator: "StringToSequenceExampleCalculator" -// input_side_packet: "SEQUENCE_EXAMPLE:sequence_example" -// output_side_packet: "STRING:serialized_sequence_example" -// } - -namespace mediapipe { -namespace tf = ::tensorflow; -namespace { -constexpr char kString[] = "STRING"; -constexpr char kSequenceExample[] = "SEQUENCE_EXAMPLE"; -} // namespace - -class StringToSequenceExampleCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; -}; - -REGISTER_CALCULATOR(StringToSequenceExampleCalculator); - -absl::Status StringToSequenceExampleCalculator::GetContract( - CalculatorContract* cc) { - if (cc->InputSidePackets().HasTag(kString)) { - cc->InputSidePackets().Tag(kString).Set(); - cc->OutputSidePackets().Tag(kSequenceExample).Set(); - } - if (cc->InputSidePackets().HasTag(kSequenceExample)) { - cc->InputSidePackets().Tag(kSequenceExample).Set(); - cc->OutputSidePackets().Tag(kString).Set(); - } - return absl::OkStatus(); -} - -absl::Status StringToSequenceExampleCalculator::Open(CalculatorContext* cc) { - if (cc->InputSidePackets().HasTag(kString)) { - auto string_value = cc->InputSidePackets().Tag(kString).Get(); - auto example = absl::make_unique(); - example->ParseFromString(string_value); - cc->OutputSidePackets() - .Tag(kSequenceExample) - .Set(mediapipe::Adopt(example.release())); - } - return absl::OkStatus(); -} - -absl::Status StringToSequenceExampleCalculator::Process(CalculatorContext* cc) { - return absl::OkStatus(); -} - -absl::Status StringToSequenceExampleCalculator::Close(CalculatorContext* cc) { - if (cc->InputSidePackets().HasTag(kSequenceExample)) { - const auto& example = - cc->InputSidePackets().Tag(kSequenceExample).Get(); - auto string_value = absl::make_unique(); - example.SerializeToString(string_value.get()); - cc->OutputSidePackets().Tag(kString).Set( - mediapipe::Adopt(string_value.release())); - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc b/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc deleted file mode 100644 index cbf494245..000000000 --- a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/framework/tensor.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; - -// Given an input Tensor (example dimensions [1, 1024, 1, 5]), it squeezes all -// dimensions with size 1, or dimensions at specific indices, producing a tensor -// containing identical data (example output dimensions [1024, 5]). -class TensorSqueezeDimensionsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) << "Need one input"; - cc->Inputs().Index(0).Set( - // Input Tensor - ); - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) << "Need one output"; - cc->Outputs().Index(0).Set( - // Output Tensor Reduced Dimensions - ); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - options_ = cc->Options(); - RET_CHECK(options_.squeeze_all_single_dims() ^ (options_.dim_size() > 0)) - << "Must specify dimensions to remove, or set squeeze_all_single_dims, " - "but not both. Received options: " - << options_.DebugString(); - if (options_.dim_size() > 0) { - remove_dims_ = - std::vector(options_.dim().begin(), options_.dim().end()); - std::sort(remove_dims_.rbegin(), remove_dims_.rend()); - remove_dims_initialized_ = true; - } - cc->SetOffset(0); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const tf::Tensor& input_tensor = cc->Inputs().Index(0).Get(); - tf::TensorShape tensor_shape = input_tensor.shape(); - if (!remove_dims_initialized_) { - // Happens iff options.squeeze_all_single_dims is set. - // Initialize remove_dims_ to all dimensions with size 1. - InitializeToRemoveAllSingletonDimensions(tensor_shape); - remove_dims_initialized_ = true; - } - for (const int dim : remove_dims_) { - RET_CHECK_GT(tensor_shape.dims(), dim) - << "Dimension " << dim - << " does not exist in input tensor with num dimensions " - << input_tensor.dims(); - RET_CHECK_EQ(tensor_shape.dim_size(dim), 1) - << "Cannot remove dimension " << dim << " with size " - << tensor_shape.dim_size(dim); - tensor_shape.RemoveDim(dim); - } - - std::unique_ptr output_tensor(new tf::Tensor); - RET_CHECK(output_tensor->CopyFrom(input_tensor, tensor_shape)); - cc->Outputs().Index(0).Add(output_tensor.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) override { - return absl::OkStatus(); - } - - private: - TensorSqueezeDimensionsCalculatorOptions options_; - std::vector remove_dims_; - bool remove_dims_initialized_; - - void InitializeToRemoveAllSingletonDimensions( - const tf::TensorShape& tensor_shape) { - const int dims = tensor_shape.dims(); - for (int i = dims - 1; i >= 0; --i) { - if (tensor_shape.dim_size(i) == 1) { - remove_dims_.push_back(i); - } - } - if (remove_dims_.empty()) { - LOG(ERROR) << "TensorSqueezeDimensionsCalculator is squeezing input with " - "no single-dimensions. Calculator will be a no-op."; - LOG(ERROR) << "Input to TensorSqueezeDimensionsCalculator has shape " - << tensor_shape.DebugString(); - } - } -}; -REGISTER_CALCULATOR(TensorSqueezeDimensionsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.proto b/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.proto deleted file mode 100644 index 87c59780d..000000000 --- a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Specifies options to TensorSqueezeDimensionsCalculator. Use this proto to -// configure which dimensions to squeeze (remove). It is only possible to remove -// dimensions of size 1. -// The fields 'squeeze_all_single_dims' and 'dim' are mutually exclusive. -message TensorSqueezeDimensionsCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorSqueezeDimensionsCalculatorOptions ext = 115619805; - } - // Remove all dimensions with size 1. - optional bool squeeze_all_single_dims = 1 [default = false]; - // Remove specific singleton dimensions. - repeated int32 dim = 2; -} diff --git a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator_test.cc deleted file mode 100644 index e3b9f7233..000000000 --- a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class TensorSqueezeDimensionsCalculatorTest : public ::testing::Test { - protected: - TensorSqueezeDimensionsCalculatorTest() { - // Initialize tensor_ with deterministic values. - tensor_shape_ = tf::TensorShape(std::vector({1, 3, 1, 3, 1})); - tensor_ = tf::Tensor(tf::DT_INT32, tensor_shape_); - auto tensor_values = tensor_.tensor(); - for (int i = 0; i < 3; ++i) { - for (int j = 0; j < 3; ++j) { - tensor_values(0, i, 0, j, 0) = i * (j + 1); - } - } - } - - std::unique_ptr runner_; - tf::TensorShape tensor_shape_; - tf::Tensor tensor_; -}; - -TEST_F(TensorSqueezeDimensionsCalculatorTest, CanSqueezeAllSingleDimensions) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorSqueezeDimensionsCalculator"); - config.add_input_stream("input_tensor"); - config.add_output_stream("output_tensor"); - CalculatorOptions options; - TensorSqueezeDimensionsCalculatorOptions* squeeze_options = - options.MutableExtension(TensorSqueezeDimensionsCalculatorOptions::ext); - squeeze_options->set_squeeze_all_single_dims(true); - *config.mutable_options() = options; - - runner_.reset(new CalculatorRunner(config)); - std::unique_ptr tensor_copy(new tf::Tensor); - EXPECT_TRUE(tensor_copy->CopyFrom(tensor_, tensor_shape_)); - const tf::int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(tensor_copy.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - const tf::TensorShape expected_shape(std::vector({3, 3})); - EXPECT_EQ(expected_shape.DebugString(), output_tensor.shape().DebugString()); - const auto tensor_values = output_tensor.tensor(); - for (int i = 0; i < 3; ++i) { - for (int j = 0; j < 3; ++j) { - const int expected_value = i * (j + 1); - EXPECT_EQ(expected_value, tensor_values(i, j)); - } - } -} - -TEST_F(TensorSqueezeDimensionsCalculatorTest, CanSqueezeSpecifiedDimensions) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorSqueezeDimensionsCalculator"); - config.add_input_stream("input_tensor"); - config.add_output_stream("output_tensor"); - CalculatorOptions options; - TensorSqueezeDimensionsCalculatorOptions* squeeze_options = - options.MutableExtension(TensorSqueezeDimensionsCalculatorOptions::ext); - squeeze_options->add_dim(0); - squeeze_options->add_dim(4); - *config.mutable_options() = options; - - runner_.reset(new CalculatorRunner(config)); - std::unique_ptr tensor_copy(new tf::Tensor); - EXPECT_TRUE(tensor_copy->CopyFrom(tensor_, tensor_shape_)); - const tf::int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(tensor_copy.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - const tf::TensorShape expected_shape(std::vector({3, 1, 3})); - EXPECT_EQ(expected_shape.DebugString(), output_tensor.shape().DebugString()); - const auto tensor_values = output_tensor.tensor(); - for (int i = 0; i < 3; ++i) { - for (int j = 0; j < 3; ++j) { - const int expected_value = i * (j + 1); - EXPECT_EQ(expected_value, tensor_values(i, 0, j)); - } - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc deleted file mode 100644 index 622e76850..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; -namespace { - -constexpr char kImage[] = "IMAGE"; -constexpr char kTensor[] = "TENSOR"; - -} // namespace - -// Input: -// Tensor of type DT_FLOAT or DT_UINT8, with values between 0-255 -// (SRGB or GRAY8). The shape can be HxWx{3,1} or simply HxW. -// -// For DT_FLOAT tensors, optionally supports a scale factor that can scale 0-1 -// value ranges to 0-255. -// -// Output: -// ImageFrame containing the values of the tensor cast as uint8 (SRGB or GRAY8) -// -// Possible extensions: support other input ranges, maybe 4D tensors. -// -// Example: -// node { -// calculator: "TensorToImageFrameCalculator" -// input_stream: "TENSOR:3d_float_tensor" -// output_stream: "IMAGE:image_frame" -// options { -// [mediapipe.TensorToImageFrameCalculatorOptions.ext] { -// scale_factor: 1.0 # set to 255.0 for [0,1] -> [0,255] scaling -// } -// } -// } -class TensorToImageFrameCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - float scale_factor_; -}; - -REGISTER_CALCULATOR(TensorToImageFrameCalculator); - -absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "One input stream must be provided."; - RET_CHECK(cc->Inputs().HasTag(kTensor)) - << "An input stream for tag: " << kTensor << " must be provided."; - cc->Inputs().Tag(kTensor).Set( - // Input Tensor. - ); - cc->Outputs().Tag(kImage).Set( - // Output ImageFrame. - ); - return absl::OkStatus(); -} - -absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) { - scale_factor_ = - cc->Options().scale_factor(); - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) { - const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get(); - int32 depth = 1; - if (input_tensor.dims() != 2) { // Depth is 1 for 2D tensors. - CHECK(3 == input_tensor.dims()) - << "Only 2 or 3-D Tensors can be converted to frames. Instead got: " - << input_tensor.dims(); - depth = input_tensor.dim_size(2); - if (depth != 1) { - RET_CHECK_EQ(depth, 3) << "Output tensor depth must be 3 or 1."; - } - } - int32 height = input_tensor.dim_size(0); - int32 width = input_tensor.dim_size(1); - auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8); - const int32 total_size = height * width * depth; - - ::std::unique_ptr output; - if (input_tensor.dtype() == tensorflow::DT_FLOAT) { - // Allocate buffer with alignments. - std::unique_ptr buffer( - new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]); - auto data = input_tensor.flat().data(); - for (int i = 0; i < total_size; ++i) { - float d = scale_factor_ * data[i]; - if (d < 0) d = 0; - if (d > 255) d = 255; - buffer[i] = d; - } - output = ::absl::make_unique(format, width, height, - width * depth, buffer.release()); - } else if (input_tensor.dtype() == tensorflow::DT_UINT8) { - if (scale_factor_ != 1.0) { - return absl::InvalidArgumentError("scale_factor_ given for uint8 tensor"); - } - // tf::Tensor has internally ref-counted buffer. The following code make the - // ImageFrame own the copied Tensor through the deleter, which increases - // the refcount of the buffer and allow us to use the shared buffer as the - // image. This allows us to create an ImageFrame object without copying - // buffer. const ImageFrame prevents the buffer from being modified later. - auto copy = new tf::Tensor(input_tensor); - output = ::absl::make_unique( - format, width, height, width * depth, copy->flat().data(), - [copy](uint8*) { delete copy; }); - } else { - return absl::InvalidArgumentError( - absl::StrCat("Expected float or uint8 tensor, received ", - DataTypeString(input_tensor.dtype()))); - } - - cc->Outputs().Tag(kImage).Add(output.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto deleted file mode 100644 index 3410068d0..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorToImageFrameCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorToImageFrameCalculatorOptions ext = 142032475; - } - - // Multiples floating point tensor outputs by this value before converting to - // uint8. This is useful for converting from range [0, 1] to [0, 255] - optional float scale_factor = 1 [default = 1.0]; -} diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc deleted file mode 100644 index 88e268907..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; -namespace { - -constexpr char kTensor[] = "TENSOR"; -constexpr char kImage[] = "IMAGE"; - -} // namespace - -template -class TensorToImageFrameCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner() { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorToImageFrameCalculator"); - config.add_input_stream("TENSOR:input_tensor"); - config.add_output_stream("IMAGE:output_image"); - runner_ = absl::make_unique(config); - } - - std::unique_ptr runner_; -}; - -using TensorToImageFrameCalculatorTestTypes = ::testing::Types; -TYPED_TEST_CASE(TensorToImageFrameCalculatorTest, - TensorToImageFrameCalculatorTestTypes); - -TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrame) { - // TYPED_TEST requires explicit "this->" - this->SetUpRunner(); - auto& runner = this->runner_; - constexpr int kWidth = 16; - constexpr int kHeight = 8; - const tf::TensorShape tensor_shape{kHeight, kWidth, 3}; - auto tensor = absl::make_unique( - tf::DataTypeToEnum::v(), tensor_shape); - auto tensor_vec = tensor->template flat().data(); - - // Writing sequence of integers as floats which we want back (as they were - // written). - for (int i = 0; i < kWidth * kHeight * 3; ++i) { - tensor_vec[i] = i % 255; - } - - const int64 time = 1234; - runner->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner->Run().ok()); - const std::vector& output_packets = - runner->Outputs().Tag(kImage).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const ImageFrame& output_image = output_packets[0].Get(); - EXPECT_EQ(ImageFormat::SRGB, output_image.Format()); - EXPECT_EQ(kWidth, output_image.Width()); - EXPECT_EQ(kHeight, output_image.Height()); - - for (int i = 0; i < kWidth * kHeight * 3; ++i) { - const uint8 pixel_value = output_image.PixelData()[i]; - EXPECT_EQ(i % 255, pixel_value); - } -} - -TYPED_TEST(TensorToImageFrameCalculatorTest, Converts3DTensorToImageFrameGray) { - this->SetUpRunner(); - auto& runner = this->runner_; - constexpr int kWidth = 16; - constexpr int kHeight = 8; - const tf::TensorShape tensor_shape{kHeight, kWidth, 1}; - auto tensor = absl::make_unique( - tf::DataTypeToEnum::v(), tensor_shape); - auto tensor_vec = tensor->template flat().data(); - - // Writing sequence of integers as floats which we want back (as they were - // written). - for (int i = 0; i < kWidth * kHeight; ++i) { - tensor_vec[i] = i % 255; - } - - const int64 time = 1234; - runner->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner->Run().ok()); - const std::vector& output_packets = - runner->Outputs().Tag(kImage).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const ImageFrame& output_image = output_packets[0].Get(); - EXPECT_EQ(ImageFormat::GRAY8, output_image.Format()); - EXPECT_EQ(kWidth, output_image.Width()); - EXPECT_EQ(kHeight, output_image.Height()); - - for (int i = 0; i < kWidth * kHeight; ++i) { - const uint8 pixel_value = output_image.PixelData()[i]; - EXPECT_EQ(i % 255, pixel_value); - } -} - -TYPED_TEST(TensorToImageFrameCalculatorTest, - Converts3DTensorToImageFrame2DGray) { - this->SetUpRunner(); - auto& runner = this->runner_; - constexpr int kWidth = 16; - constexpr int kHeight = 8; - const tf::TensorShape tensor_shape{kHeight, kWidth}; - auto tensor = absl::make_unique( - tf::DataTypeToEnum::v(), tensor_shape); - auto tensor_vec = tensor->template flat().data(); - - // Writing sequence of integers as floats which we want back (as they were - // written). - for (int i = 0; i < kWidth * kHeight; ++i) { - tensor_vec[i] = i % 255; - } - - const int64 time = 1234; - runner->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner->Run().ok()); - const std::vector& output_packets = - runner->Outputs().Tag(kImage).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const ImageFrame& output_image = output_packets[0].Get(); - EXPECT_EQ(ImageFormat::GRAY8, output_image.Format()); - EXPECT_EQ(kWidth, output_image.Width()); - EXPECT_EQ(kHeight, output_image.Height()); - - for (int i = 0; i < kWidth * kHeight; ++i) { - const uint8 pixel_value = output_image.PixelData()[i]; - EXPECT_EQ(i % 255, pixel_value); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc deleted file mode 100644 index 85955c43b..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Calculator converts from one-dimensional Tensor of DT_FLOAT to Matrix -// OR from (batched) two-dimensional Tensor of DT_FLOAT to Matrix. - -#include "mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; -namespace { - -constexpr char kMatrix[] = "MATRIX"; -constexpr char kTensor[] = "TENSOR"; -constexpr char kReference[] = "REFERENCE"; - -absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, - TimeSeriesHeader* header) { - CHECK(header); - if (header_packet.IsEmpty()) { - return absl::UnknownError("No header found."); - } - if (!header_packet.ValidateAsType().ok()) { - return absl::UnknownError("Packet does not contain TimeSeriesHeader."); - } - *header = header_packet.Get(); - if (header->has_sample_rate() && header->sample_rate() >= 0 && - header->has_num_channels() && header->num_channels() >= 0) { - return absl::OkStatus(); - } else { - std::string error_message = - "TimeSeriesHeader is missing necessary fields: " - "sample_rate or num_channels, or one of their values is negative. "; -#ifndef MEDIAPIPE_MOBILE - absl::StrAppend(&error_message, "Got header:\n", - header->ShortDebugString()); -#endif - return absl::InvalidArgumentError(error_message); - } -} - -} // namespace - -// Converts a 1-D or a 2-D Tensor into a 2 dimensional Matrix. -// Input: -// -- 1-D or 2-D Tensor -// Output: -// -- Matrix with the same values as the Tensor -// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape. -// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape. -// -// Example Config -// node: { -// calculator: "TensorToMatrixCalculator" -// input_stream: "TENSOR:tensor" -// output_stream: "MATRIX:matrix" -// } -// -// -// This calculator produces a TimeSeriesHeader header on its output stream iff -// an input stream is supplied with the REFERENCE tag and that stream has a -// header of type TimeSeriesHeader. This header is modified in two ways: -// - the sample_rate is set to the packet rate of the REFERENCE stream (which -// must have a packet_rate defined in its header). This is under the -// assumption that the packets on the reference stream, input stream, and -// output stream are in a 1:1 correspondence, and that the output packets are -// 1-D column vectors that represent a single sample of output. -// - the TimeSeriesHeader overrides specified in the calculator options are -// then applied, which can override the sample_rate field. -// If the REFERENCE stream is supplied, then the TimeSeriesHeader is verified on -// the input data when it arrives in Process(). In particular, if the header -// states that we produce a 1xD column vector, the input tensor must also be 1xD -// -// Example Config -// node: { -// calculator: "TensorToMatrixCalculator" -// input_stream: "TENSOR:tensor" -// input_stream: "REFERENCE:reference_matrix" -// output_stream: "MATRIX:matrix" -// options { -// [mediapipe.TensorToMatrixCalculatorOptions.ext] { -// time_series_header_overrides { -// num_channels: 128 -// } -// } -// } -// } -class TensorToMatrixCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - // Store header information so that we can verify the inputs in process(). - TimeSeriesHeader header_; -}; -REGISTER_CALCULATOR(TensorToMatrixCalculator); - -absl::Status TensorToMatrixCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_LE(cc->Inputs().NumEntries(), 2) - << "Only one or two input streams are supported."; - RET_CHECK_GT(cc->Inputs().NumEntries(), 0) - << "At least one input stream must be provided."; - RET_CHECK(cc->Inputs().HasTag(kTensor)) - << "An input stream for tag: " << kTensor << " must be provided."; - cc->Inputs().Tag(kTensor).Set( - // Input Tensor. - ); - if (cc->Inputs().NumEntries() == 2) { - RET_CHECK(cc->Inputs().HasTag(kReference)) - << "An input stream for tag: " << kReference - << " must be provided when" - " providing two inputs."; - cc->Inputs() - .Tag(kReference) - .Set( - // A reference stream for the header. - ); - } - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Tag(kMatrix).Set( - // Output Matrix. - ); - return absl::OkStatus(); -} - -absl::Status TensorToMatrixCalculator::Open(CalculatorContext* cc) { - auto input_header = absl::make_unique(); - absl::Status header_status; - if (cc->Inputs().HasTag(kReference)) { - header_status = FillTimeSeriesHeaderIfValid( - cc->Inputs().Tag(kReference).Header(), input_header.get()); - } - if (header_status.ok()) { - if (cc->Options() - .has_time_series_header_overrides()) { - // This only supports a single sample per packet for now, so we hardcode - // the sample_rate based on the packet_rate of the REFERENCE and fail - // if we cannot. - const TimeSeriesHeader& override_header = - cc->Options() - .time_series_header_overrides(); - input_header->MergeFrom(override_header); - RET_CHECK(input_header->has_packet_rate()) - << "The TimeSeriesHeader.packet_rate must be set."; - if (!override_header.has_sample_rate()) { - RET_CHECK_EQ(input_header->num_samples(), 1) - << "Currently the time series can only output single samples."; - input_header->set_sample_rate(input_header->packet_rate()); - } - } - header_ = *input_header; - cc->Outputs().Tag(kMatrix).SetHeader(Adopt(input_header.release())); - } - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) { - // Verify that each reference stream packet corresponds to a tensor packet - // otherwise the header information is invalid. If we don't have a reference - // stream, Process() is only called when we have an input tensor and this is - // always True. - RET_CHECK(cc->Inputs().HasTag(kTensor)) - << "Tensor stream not available at same timestamp as the reference " - "stream."; - RET_CHECK(!cc->Inputs().Tag(kTensor).IsEmpty()) << "Tensor stream is empty."; - RET_CHECK_OK(cc->Inputs().Tag(kTensor).Value().ValidateAsType()) - << "Tensor stream packet does not contain a Tensor."; - - const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get(); - CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims()) - << "Only 1-D or 2-D Tensors can be converted to matrices."; - const int32 length = input_tensor.dim_size(input_tensor.dims() - 1); - const int32 width = (1 == input_tensor.dims()) ? 1 : input_tensor.dim_size(0); - if (header_.has_num_channels()) { - RET_CHECK_EQ(length, header_.num_channels()) - << "The number of channels at runtime does not match the header."; - } - if (header_.has_num_samples()) { - RET_CHECK_EQ(width, header_.num_samples()) - << "The number of samples at runtime does not match the header."; - } - auto output = absl::make_unique(width, length); - *output = - Eigen::MatrixXf::Map(input_tensor.flat().data(), length, width); - cc->Outputs().Tag(kMatrix).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.proto b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.proto deleted file mode 100644 index e8647700c..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.proto +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/framework/formats/time_series_header.proto"; - -message TensorToMatrixCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorToMatrixCalculatorOptions ext = 136654056; - } - - // Any values present in this TimeSeriesHeader override the values in the - // header from the reference stream if the reference stream is used. - // The most common fields to override are the num_channels field which - // typically correspond to the dimensionality of an output embedding and - // the num_samples field which is generally 1 for an embedding. - // If the sampling_rate is not specified in the time_series_header, then - // the packet_rate from the reference stream will be used as the sampling_rate - // which assumes that num_samples is 1. - optional TimeSeriesHeader time_series_header_overrides = 1; -} diff --git a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator_test.cc deleted file mode 100644 index fce24b9b9..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator_test.cc +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; -namespace { - -constexpr char kMatrix[] = "MATRIX"; -constexpr char kTensor[] = "TENSOR"; - -} // namespace - -class TensorToMatrixCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner() { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorToMatrixCalculator"); - config.add_input_stream("TENSOR:input_tensor"); - config.add_output_stream("MATRIX:output_matrix"); - runner_ = absl::make_unique(config); - } - - // Creates a reference stream and sets num_channels and num_samples if > 0. - void SetUpRunnerWithReference(int channels, int samples, - int override_channels, bool include_rate) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorToMatrixCalculator"); - config.add_input_stream("TENSOR:input_tensor"); - config.add_input_stream("REFERENCE:reference"); - config.add_output_stream("MATRIX:output_matrix"); - if (override_channels > 0) { - config.mutable_options() - ->MutableExtension(TensorToMatrixCalculatorOptions::ext) - ->mutable_time_series_header_overrides() - ->set_num_channels(override_channels); - } - runner_ = absl::make_unique(config); - - auto header = absl::make_unique(); - header->set_sample_rate(1.0); - if (channels > 0) { - header->set_num_channels(channels); - } - if (samples > 0) { - header->set_num_samples(samples); - } - if (include_rate) { - header->set_packet_rate(1.0); - } - runner_->MutableInputs()->Tag("REFERENCE").header = Adopt(header.release()); - } - - std::unique_ptr runner_; -}; - -TEST_F(TensorToMatrixCalculatorTest, Converts1DTensorToMatrix) { - // This test converts a 1 Dimensional Tensor of length M to a Matrix of Mx1. - SetUpRunner(); - const tf::TensorShape tensor_shape(std::vector{5}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto tensor_vec = tensor->vec(); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - tensor_vec(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Tag(kMatrix).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const Matrix& output_matrix = output_packets[0].Get(); - EXPECT_EQ(5, output_matrix.rows()); - for (int i = 0; i < 5; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_matrix(i, 0)); - } -} - -TEST_F(TensorToMatrixCalculatorTest, Converts2DTensorofWidthOneToMatrix) { - // This test converts a 2 Dimensional Tensor of shape 1xM to a Matrix of Mx1. - SetUpRunner(); - const tf::TensorShape tensor_shape(std::vector({1, 4})); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto slice = tensor->Slice(0, 1).flat(); - for (int i = 0; i < 4; ++i) { - slice(i) = static_cast(1 << i); - } - const int64 time = 1234; - runner_->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Tag(kMatrix).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const Matrix& output_matrix = output_packets[0].Get(); - ASSERT_EQ(1, output_matrix.cols()); - EXPECT_EQ(4, output_matrix.rows()); - for (int i = 0; i < 4; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_matrix(i, 0)); - } -} - -TEST_F(TensorToMatrixCalculatorTest, Converts2DTensorToMatrix) { - // This test converts a 2 Dimensional Tensor of shape NxM to a Matrix of MxN. - SetUpRunner(); - const tf::TensorShape tensor_shape(std::vector({3, 4})); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto slice = tensor->Slice(0, 1).flat(); - for (int i = 0; i < 3; ++i) { - for (int j = 0; j < 4; ++j) { - slice(i * 4 + j) = static_cast(i * j); - } - } - const int64 time = 1234; - runner_->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Tag(kMatrix).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const Matrix& output_matrix = output_packets[0].Get(); - ASSERT_EQ(3, output_matrix.cols()); - EXPECT_EQ(4, output_matrix.rows()); - for (int i = 0; i < 4; ++i) { - for (int j = 0; j < 3; ++j) { - const float expected = static_cast(i * j); - EXPECT_EQ(expected, output_matrix(i, j)); - } - } -} - -TEST_F(TensorToMatrixCalculatorTest, ConvertsWithReferenceTimeSeriesHeader) { - // This test converts a 1 Dimensional Tensor of length M to a Matrix of Mx1. - SetUpRunnerWithReference(5, 1, -1, true); - const tf::TensorShape tensor_shape(std::vector{5}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto tensor_vec = tensor->vec(); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - tensor_vec(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Tag(kMatrix).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const Matrix& output_matrix = output_packets[0].Get(); - EXPECT_EQ(5, output_matrix.rows()); - for (int i = 0; i < 5; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_matrix(i, 0)); - } - - const TimeSeriesHeader& output_header = - runner_->Outputs().Tag(kMatrix).header.Get(); - EXPECT_EQ(output_header.num_channels(), 5); -} - -TEST_F(TensorToMatrixCalculatorTest, TimeSeriesOverridesWork) { - // This test converts a 1 Dimensional Tensor of length M to a Matrix of Mx1. - SetUpRunnerWithReference(7, 1, 5, true); - const tf::TensorShape tensor_shape(std::vector{5}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto tensor_vec = tensor->vec(); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - tensor_vec(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Tag(kTensor).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Tag(kMatrix).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const Matrix& output_matrix = output_packets[0].Get(); - EXPECT_EQ(5, output_matrix.rows()); - for (int i = 0; i < 5; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_matrix(i, 0)); - } - - const TimeSeriesHeader& output_header = - runner_->Outputs().Tag(kMatrix).header.Get(); - EXPECT_EQ(output_header.num_channels(), 5); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator.cc deleted file mode 100644 index cd807b87b..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector -// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector. - -#include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; - -class TensorToVectorFloatCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - TensorToVectorFloatCalculatorOptions options_; -}; -REGISTER_CALCULATOR(TensorToVectorFloatCalculator); - -absl::Status TensorToVectorFloatCalculator::GetContract( - CalculatorContract* cc) { - // Start with only one input packet. - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - cc->Inputs().Index(0).Set( - // Input Tensor - ); - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - const auto& options = cc->Options(); - if (options.tensor_is_2d()) { - RET_CHECK(!options.flatten_nd()); - cc->Outputs().Index(0).Set>>( - /* "Output vector>." */); - } else { - cc->Outputs().Index(0).Set>( - // Output vector. - ); - } - return absl::OkStatus(); -} - -absl::Status TensorToVectorFloatCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - - // Inform mediapipe that this calculator produces an output at time t for - // each input received at time t (i.e. this calculator does not buffer - // inputs). This enables mediapipe to propagate time of arrival estimates in - // mediapipe graphs through this calculator. - cc->SetOffset(/*offset=*/0); - - return absl::OkStatus(); -} - -absl::Status TensorToVectorFloatCalculator::Process(CalculatorContext* cc) { - const tf::Tensor& input_tensor = - cc->Inputs().Index(0).Value().Get(); - RET_CHECK(tf::DT_FLOAT == input_tensor.dtype()) - << "expected DT_FLOAT input but got " - << tensorflow::DataTypeString(input_tensor.dtype()); - - if (options_.tensor_is_2d()) { - RET_CHECK(2 == input_tensor.dims()) - << "Expected 2-dimensional Tensor, but the tensor shape is: " - << input_tensor.shape().DebugString(); - auto output = absl::make_unique>>( - input_tensor.dim_size(0), std::vector(input_tensor.dim_size(1))); - for (int i = 0; i < input_tensor.dim_size(0); ++i) { - auto& instance_output = output->at(i); - const auto& slice = input_tensor.Slice(i, i + 1).unaligned_flat(); - for (int j = 0; j < input_tensor.dim_size(1); ++j) { - instance_output.at(j) = slice(j); - } - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else { - if (!options_.flatten_nd()) { - RET_CHECK(1 == input_tensor.dims()) - << "`flatten_nd` is not set. Expected 1-dimensional Tensor, but the " - << "tensor shape is: " << input_tensor.shape().DebugString(); - } - auto output = - absl::make_unique>(input_tensor.NumElements()); - const auto& tensor_values = input_tensor.flat(); - for (int i = 0; i < input_tensor.NumElements(); ++i) { - output->at(i) = tensor_values(i); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.proto b/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.proto deleted file mode 100644 index c9aa67f52..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorToVectorFloatCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorToVectorFloatCalculatorOptions ext = 120862965; - } - - // If true, unpack a 2d tensor (matrix) into a vector>. If - // false, convert a 1d tensor (vector) into a vector. - optional bool tensor_is_2d = 1 [default = false]; - - // If true, an N-D tensor will be flattened to a vector. This is - // exclusive with tensor_is_2d. - optional bool flatten_nd = 2 [default = false]; -} diff --git a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_test.cc deleted file mode 100644 index 69d3af60a..000000000 --- a/mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_test.cc +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class TensorToVectorFloatCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorToVectorFloatCalculator"); - config.add_input_stream("input_tensor"); - config.add_output_stream("output_tensor"); - auto options = config.mutable_options()->MutableExtension( - TensorToVectorFloatCalculatorOptions::ext); - options->set_tensor_is_2d(tensor_is_2d); - options->set_flatten_nd(flatten_nd); - runner_ = absl::make_unique(config); - } - - std::unique_ptr runner_; -}; - -TEST_F(TensorToVectorFloatCalculatorTest, ConvertsToVectorFloat) { - SetUpRunner(false, false); - const tf::TensorShape tensor_shape(std::vector{5}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto tensor_vec = tensor->vec(); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - tensor_vec(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const std::vector& output_vector = - output_packets[0].Get>(); - - EXPECT_EQ(5, output_vector.size()); - for (int i = 0; i < 5; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_vector[i]); - } -} - -TEST_F(TensorToVectorFloatCalculatorTest, ConvertsBatchedToVectorVectorFloat) { - SetUpRunner(true, false); - const tf::TensorShape tensor_shape(std::vector{1, 5}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto slice = tensor->Slice(0, 1).flat(); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - slice(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const std::vector>& output_vectors = - output_packets[0].Get>>(); - ASSERT_EQ(1, output_vectors.size()); - const std::vector& output_vector = output_vectors[0]; - EXPECT_EQ(5, output_vector.size()); - for (int i = 0; i < 5; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_vector[i]); - } -} - -TEST_F(TensorToVectorFloatCalculatorTest, FlattenShouldTakeAllDimensions) { - SetUpRunner(false, true); - const tf::TensorShape tensor_shape(std::vector{2, 2, 2}); - auto tensor = absl::make_unique(tf::DT_FLOAT, tensor_shape); - auto slice = tensor->flat(); - for (int i = 0; i < 2 * 2 * 2; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - slice(i) = static_cast(1 << i); - } - - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(tensor.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const std::vector& output_vector = - output_packets[0].Get>(); - EXPECT_EQ(2 * 2 * 2, output_vector.size()); - for (int i = 0; i < 2 * 2 * 2; ++i) { - const float expected = static_cast(1 << i); - EXPECT_EQ(expected, output_vector[i]); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc deleted file mode 100644 index 625612c17..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ /dev/null @@ -1,647 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -#include "absl/base/thread_annotations.h" -#include "absl/memory/memory.h" -#include "absl/strings/str_split.h" -#include "absl/synchronization/mutex.h" -#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/map_util.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/status_util.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_util.h" - -#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__) -#include "tensorflow/core/profiler/lib/traceme.h" -#endif - -namespace tf = ::tensorflow; - -namespace mediapipe { - -namespace { -// This is a simple implementation of a semaphore using standard C++ libraries. -// It is supposed to be used only by TensorflowInferenceCalculator to throttle -// the concurrent calls of Tensorflow Session::Run. This is useful when multiple -// threads execute the graph (e.g. in a mapreduce type of job) but not to -// overload GPU/TPU/... -class SimpleSemaphore { - public: - explicit SimpleSemaphore(uint32 initial_count) : count_(initial_count) {} - SimpleSemaphore(const SimpleSemaphore&) = delete; - SimpleSemaphore(SimpleSemaphore&&) = delete; - - // Acquires the semaphore by certain amount. - void Acquire(uint32 amount) { - mutex_.Lock(); - while (count_ < amount) { - cond_.Wait(&mutex_); - } - count_ -= amount; - mutex_.Unlock(); - } - - // Releases the semaphore by certain amount. - void Release(uint32 amount) { - mutex_.Lock(); - count_ += amount; - cond_.SignalAll(); - mutex_.Unlock(); - } - - private: - uint32 count_; - absl::Mutex mutex_; - absl::CondVar cond_; -}; - -class InferenceState { - public: - InferenceState() : input_tensor_batches_(), batch_timestamps_() {} - // A mapping between stream tags and the tensors we are collecting as a - // batch. - std::map> input_tensor_batches_; - // The timestamps that go into a batch. - std::vector batch_timestamps_; -}; - -} // namespace - -// This calculator performs inference on a trained TensorFlow model. -// -// TensorFlow Sessions can be created from checkpoint paths, frozen models, or -// the SavedModel system. See the TensorFlowSessionFrom* packet generators for -// details. Each of these methods defines a mapping between MediaPipe streams -// and TensorFlow tensors. All of this information is passed in as an -// input_side_packet. -// -// The input and output streams are TensorFlow tensors labeled by tags. The tags -// for the streams are matched to feeds and fetchs in a TensorFlow session using -// a named_signature.generic_signature in the ModelManifest. The -// generic_signature is used as key-value pairs between the MediaPipe tag and -// the TensorFlow tensor. The signature_name in the options proto determines -// which named_signature is used. The keys in the generic_signature must be -// valid MediaPipe tags ([A-Z0-9_]*, no lowercase or special characters). All of -// the tensors corresponding to tags in the signature for input_streams are fed -// to the model and for output_streams the tensors are fetched from the model. -// -// Other calculators are used to convert data to and from tensors, this op only -// handles the TensorFlow session and batching. Batching occurs by concatenating -// input tensors along the 0th dimension across timestamps. If the 0th dimension -// is not a batch dimension, this calculator will add a 0th dimension by -// default. Setting add_batch_dim_to_tensors to false disables the dimension -// addition. Once batch_size inputs have been provided, the batch will be run -// and the output tensors sent out on the output streams with timestamps -// corresponding to the input stream packets. Setting the batch_size to 1 -// completely disables batching, but is indepdent of add_batch_dim_to_tensors. -// -// The TensorFlowInferenceCalculator also support feeding states recurrently for -// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the -// recurrent tensors. Initializing the recurrent state can be handled by the -// GraphTensorsPacketGenerator. -// -// The calculator updates two Counters to report timing information: -// ---TotalTimeUsecs = Total time spent running inference (in usecs), -// ---TotalProcessedTimestamps = # of instances processed -// (approximately batches processed * batch_size), -// where is replaced with CalculatorGraphConfig::Node::name() if it -// exists, or with TensorFlowInferenceCalculator if the name is not set. The -// name must be set for timing information to be instance-specific in graphs -// with multiple TensorFlowInferenceCalculators. -// -// Example config: -// packet_generator { -// packet_generator: "TensorFlowSessionFromSavedModelGenerator" -// output_side_packet: "tensorflow_session" -// options { -// [mediapipe.TensorFlowSessionFromSavedModelGeneratorOptions.ext]: { -// saved_model_path: "/path/to/saved/model" -// signature_name: "mediapipe" -// } -// } -// } -// node { -// calculator: "TensorFlowInferenceCalculator" -// input_stream: "IMAGES:image_tensors_keyed_in_signature_by_tag" -// input_stream: "AUDIO:audio_tensors_keyed_in_signature_by_tag" -// output_stream: "LABELS:softmax_tensor_keyed_in_signature_by_tag" -// input_side_packet: "SESSION:tensorflow_session" -// } -// -// Where the input and output streams are treated as Packet and -// the mediapipe_signature has tensor bindings between "IMAGES", "AUDIO", and -// "LABELS" and their respective tensors exported to /path/to/bundle. For an -// example of how this model was exported, see -// tensorflow_inference_test_graph_generator.py -// -// It is possible to use a GraphDef proto that was not exported by exporter (i.e -// without MetaGraph with bindings). Such GraphDef could contain all of its -// parameters in-lined (for example, it can be the output of freeze_graph.py). -// To instantiate a TensorFlow model from a GraphDef file, replace the -// packet_factory above with TensorFlowSessionFromFrozenGraphGenerator: -// -// packet_generator { -// packet_generator: "TensorFlowSessionFromFrozenGraphGenerator" -// output_side_packet: "SESSION:tensorflow_session" -// options { -// [mediapipe.TensorFlowSessionFromFrozenGraphGeneratorOptions.ext]: { -// graph_proto_path: "[PATH]" -// tag_to_tensor_names { -// key: "JPG_STRING" -// value: "input:0" -// } -// tag_to_tensor_names { -// key: "SOFTMAX" -// value: "softmax:0" -// } -// } -// } -// } -// -// It is also possible to use a GraphDef proto and checkpoint file that have not -// been frozen. This can be used to load graphs directly as they have been -// written from training. However, it is more brittle and you are encouraged to -// use a one of the more perminent formats described above. To instantiate a -// TensorFlow model from a GraphDef file and checkpoint, replace the -// packet_factory above with TensorFlowSessionFromModelCheckpointGenerator: -// -// packet_generator { -// packet_generator: "TensorFlowSessionFromModelCheckpointGenerator" -// output_side_packet: "SESSION:tensorflow_session" -// options { -// [mediapipe.TensorFlowSessionFromModelCheckpointGeneratorOptions.ext]: { -// graph_proto_path: "[PATH]" -// model_options { -// checkpoint_path: "[PATH2]" -// } -// tag_to_tensor_names { -// key: "JPG_STRING" -// value: "input:0" -// } -// tag_to_tensor_names { -// key: "SOFTMAX" -// value: "softmax:0" -// } -// } -// } -// } -class TensorFlowInferenceCalculator : public CalculatorBase { - public: - // Counters for recording timing information. The actual names have the value - // of CalculatorGraphConfig::Node::name() prepended. - static constexpr char kTotalUsecsCounterSuffix[] = "TotalTimeUsecs"; - static constexpr char kTotalProcessedTimestampsCounterSuffix[] = - "TotalProcessedTimestamps"; - static constexpr char kTotalSessionRunsTimeUsecsCounterSuffix[] = - "TotalSessionRunsTimeUsecs"; - static constexpr char kTotalNumSessionRunsCounterSuffix[] = - "TotalNumSessionRuns"; - - TensorFlowInferenceCalculator() : session_(nullptr) { - clock_ = std::unique_ptr( - mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock()); - } - - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - RET_CHECK(!cc->Inputs().GetTags().empty()); - for (const std::string& tag : cc->Inputs().GetTags()) { - // The tensorflow::Tensor with the tag equal to the graph node. May - // have a TimeSeriesHeader if all present TimeSeriesHeaders match. - if (!options.batched_input()) { - cc->Inputs().Tag(tag).Set(); - } else { - cc->Inputs().Tag(tag).Set>(); - } - } - RET_CHECK(!cc->Outputs().GetTags().empty()); - for (const std::string& tag : cc->Outputs().GetTags()) { - // The tensorflow::Tensor with tag equal to the graph node to - // output. Any TimeSeriesHeader from the inputs will be forwarded - // with channels set to 0. - cc->Outputs().Tag(tag).Set(); - } - // A mediapipe::TensorFlowSession with a model loaded and ready for use. - // For this calculator it must include a tag_to_tensor_map. - cc->InputSidePackets().Tag("SESSION").Set(); - if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS")) { - cc->InputSidePackets() - .Tag("RECURRENT_INIT_TENSORS") - .Set>>(); - } - return absl::OkStatus(); - } - - std::unique_ptr CreateInferenceState(CalculatorContext* cc) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { - std::unique_ptr inference_state = - absl::make_unique(); - if (cc->InputSidePackets().HasTag("RECURRENT_INIT_TENSORS") && - !cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS").IsEmpty()) { - std::map* init_tensor_map; - init_tensor_map = GetFromUniquePtr>( - cc->InputSidePackets().Tag("RECURRENT_INIT_TENSORS")); - for (const auto& p : *init_tensor_map) { - inference_state->input_tensor_batches_[p.first].emplace_back(p.second); - } - } - return inference_state; - } - - absl::Status Open(CalculatorContext* cc) override { - options_ = cc->Options(); - - RET_CHECK(cc->InputSidePackets().HasTag("SESSION")); - session_ = cc->InputSidePackets() - .Tag("SESSION") - .Get() - .session.get(); - tag_to_tensor_map_ = cc->InputSidePackets() - .Tag("SESSION") - .Get() - .tag_to_tensor_map; - - // Validate and store the recurrent tags - RET_CHECK(options_.has_batch_size()); - RET_CHECK(options_.batch_size() == 1 || - options_.recurrent_tag_pair().empty()) - << "To use recurrent_tag_pairs, batch_size must be 1."; - for (const auto& tag_pair : options_.recurrent_tag_pair()) { - const std::vector tags = absl::StrSplit(tag_pair, ':'); - RET_CHECK_EQ(tags.size(), 2) - << "recurrent_tag_pair must be a colon " - "separated std::string with two components: " - << tag_pair; - RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[0])) - << "Can't find tag '" << tags[0] << "' in signature " - << options_.signature_name(); - RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tags[1])) - << "Can't find tag '" << tags[1] << "' in signature " - << options_.signature_name(); - recurrent_feed_tags_.insert(tags[0]); - recurrent_fetch_tags_to_feed_tags_[tags[1]] = tags[0]; - } - - // Check that all tags are present in this signature bound to tensors. - for (const std::string& tag : cc->Inputs().GetTags()) { - RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag)) - << "Can't find tag '" << tag << "' in signature " - << options_.signature_name(); - } - for (const std::string& tag : cc->Outputs().GetTags()) { - RET_CHECK(mediapipe::ContainsKey(tag_to_tensor_map_, tag)) - << "Can't find tag '" << tag << "' in signature " - << options_.signature_name(); - } - - { - absl::WriterMutexLock l(&mutex_); - inference_state_ = std::unique_ptr(); - } - - if (options_.batch_size() == 1 || options_.batched_input()) { - cc->SetOffset(0); - } - - return absl::OkStatus(); - } - - // Adds a batch dimension to the input tensor if specified in the calculator - // options. - absl::Status AddBatchDimension(tf::Tensor* input_tensor) { - if (options_.add_batch_dim_to_tensors()) { - tf::TensorShape new_shape(input_tensor->shape()); - new_shape.InsertDim(0, 1); - RET_CHECK(input_tensor->CopyFrom(*input_tensor, new_shape)) - << "Could not add 0th dimension to tensor without changing its shape." - << " Current shape: " << input_tensor->shape().DebugString(); - } - return absl::OkStatus(); - } - - absl::Status AggregateTensorPacket( - const std::string& tag_name, const Packet& packet, - std::map>* - input_tensors_by_tag_by_timestamp, - InferenceState* inference_state) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { - tf::Tensor input_tensor(packet.Get()); - RET_CHECK_OK(AddBatchDimension(&input_tensor)); - if (mediapipe::ContainsKey(recurrent_feed_tags_, tag_name)) { - // If we receive an input on a recurrent tag, override the state. - // It's OK to override the global state because there is just one - // input stream allowed for recurrent tensors. - inference_state_->input_tensor_batches_[tag_name].clear(); - } - (*input_tensors_by_tag_by_timestamp)[packet.Timestamp()].insert( - std::make_pair(tag_name, input_tensor)); - return absl::OkStatus(); - } - - // Removes the batch dimension of the output tensor if specified in the - // calculator options. - absl::Status RemoveBatchDimension(tf::Tensor* output_tensor) { - if (options_.add_batch_dim_to_tensors()) { - tf::TensorShape new_shape(output_tensor->shape()); - new_shape.RemoveDim(0); - RET_CHECK(output_tensor->CopyFrom(*output_tensor, new_shape)) - << "Could not remove 0th dimension from tensor without changing its " - << "shape. Current shape: " << output_tensor->shape().DebugString() - << " (The expected first dimension is 1 for a batch element.)"; - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - std::unique_ptr inference_state_to_process; - { - absl::WriterMutexLock l(&mutex_); - if (inference_state_ == nullptr) { - inference_state_ = CreateInferenceState(cc); - } - std::map> - input_tensors_by_tag_by_timestamp; - for (const std::string& tag_as_node_name : cc->Inputs().GetTags()) { - if (cc->Inputs().Tag(tag_as_node_name).IsEmpty()) { - // Recurrent tensors can be empty. - if (!mediapipe::ContainsKey(recurrent_feed_tags_, tag_as_node_name)) { - if (options_.skip_on_missing_features()) { - return absl::OkStatus(); - } else { - return absl::InvalidArgumentError(absl::StrCat( - "Tag ", tag_as_node_name, - " not present at timestamp: ", cc->InputTimestamp().Value())); - } - } - } else if (options_.batched_input()) { - const auto& tensor_packets = - cc->Inputs().Tag(tag_as_node_name).Get>(); - if (tensor_packets.size() > options_.batch_size()) { - return absl::InvalidArgumentError(absl::StrCat( - "Batch for tag ", tag_as_node_name, - " has more packets than batch capacity. batch_size: ", - options_.batch_size(), " packets: ", tensor_packets.size())); - } - for (const auto& packet : tensor_packets) { - RET_CHECK_OK(AggregateTensorPacket( - tag_as_node_name, packet, &input_tensors_by_tag_by_timestamp, - inference_state_.get())); - } - } else { - RET_CHECK_OK(AggregateTensorPacket( - tag_as_node_name, cc->Inputs().Tag(tag_as_node_name).Value(), - &input_tensors_by_tag_by_timestamp, inference_state_.get())); - } - } - for (const auto& timestamp_and_input_tensors_by_tag : - input_tensors_by_tag_by_timestamp) { - inference_state_->batch_timestamps_.emplace_back( - timestamp_and_input_tensors_by_tag.first); - for (const auto& input_tensor_and_tag : - timestamp_and_input_tensors_by_tag.second) { - inference_state_->input_tensor_batches_[input_tensor_and_tag.first] - .emplace_back(input_tensor_and_tag.second); - } - } - if (inference_state_->batch_timestamps_.size() == options_.batch_size() || - options_.batched_input()) { - inference_state_to_process = std::move(inference_state_); - inference_state_ = std::unique_ptr(); - } - } - - if (inference_state_to_process) { - MP_RETURN_IF_ERROR( - OutputBatch(cc, std::move(inference_state_to_process))); - } - - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) override { - std::unique_ptr inference_state_to_process = nullptr; - { - absl::WriterMutexLock l(&mutex_); - if (cc->GraphStatus().ok() && inference_state_ != nullptr && - !inference_state_->batch_timestamps_.empty()) { - inference_state_to_process = std::move(inference_state_); - inference_state_ = std::unique_ptr(); - } - } - if (inference_state_to_process) { - MP_RETURN_IF_ERROR( - OutputBatch(cc, std::move(inference_state_to_process))); - } - return absl::OkStatus(); - } - - // When a batch of input tensors is ready to be run, runs TensorFlow and - // outputs the output tensors. The output tensors have timestamps matching - // the input tensor that formed that batch element. Any requested - // batch_dimension is added and removed. This code takes advantage of the fact - // that copying a tensor shares the same reference-counted, heap allocated - // memory buffer. Therefore, copies are cheap and should not cause the memory - // buffer to fall out of scope. In contrast, concat is only used where - // necessary. - absl::Status OutputBatch(CalculatorContext* cc, - std::unique_ptr inference_state) { - const int64 start_time = absl::ToUnixMicros(clock_->TimeNow()); - std::vector> input_tensors; - - for (auto& keyed_tensors : inference_state->input_tensor_batches_) { - if (options_.batch_size() == 1) { - // Short circuit to avoid the cost of deep copying tensors in concat. - if (!keyed_tensors.second.empty()) { - input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first], - keyed_tensors.second[0]); - } else { - // The input buffer can be empty for recurrent tensors. - RET_CHECK( - mediapipe::ContainsKey(recurrent_feed_tags_, keyed_tensors.first)) - << "A non-recurrent tensor does not have an input: " - << keyed_tensors.first; - } - } else { - // Pad by replicating the first tens or, then ignore the values. - keyed_tensors.second.resize(options_.batch_size()); - std::fill(keyed_tensors.second.begin() + - inference_state->batch_timestamps_.size(), - keyed_tensors.second.end(), keyed_tensors.second[0]); - tf::Tensor concated; - const tf::Status concat_status = - tf::tensor::Concat(keyed_tensors.second, &concated); - CHECK(concat_status.ok()) << concat_status.ToString(); - input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first], - concated); - } - } - inference_state->input_tensor_batches_.clear(); - std::vector output_tensor_names; - std::vector output_name_in_signature; - for (const std::string& tag : cc->Outputs().GetTags()) { - output_tensor_names.emplace_back(tag_to_tensor_map_[tag]); - output_name_in_signature.emplace_back(tag); - } - for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) { - // Ensure that we always fetch the recurrent state tensors. - if (std::find(output_name_in_signature.begin(), - output_name_in_signature.end(), - tag_pair.first) == output_name_in_signature.end()) { - output_tensor_names.emplace_back(tag_to_tensor_map_[tag_pair.first]); - output_name_in_signature.emplace_back(tag_pair.first); - } - } - std::vector outputs; - - SimpleSemaphore* session_run_throttle = nullptr; - if (options_.max_concurrent_session_runs() > 0) { - session_run_throttle = - get_session_run_throttle(options_.max_concurrent_session_runs()); - session_run_throttle->Acquire(1); - } - const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow()); - tf::Status tf_status; - { -#if !defined(MEDIAPIPE_MOBILE) && !defined(__APPLE__) - tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName())); -#endif - tf_status = session_->Run(input_tensors, output_tensor_names, - {} /* target_node_names */, &outputs); - } - - if (session_run_throttle != nullptr) { - session_run_throttle->Release(1); - } - - // RET_CHECK on the tf::Status object itself in order to print an - // informative error message. - RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString(); - - const int64 run_end_time = absl::ToUnixMicros(clock_->TimeNow()); - cc->GetCounter(kTotalSessionRunsTimeUsecsCounterSuffix) - ->IncrementBy(run_end_time - run_start_time); - cc->GetCounter(kTotalNumSessionRunsCounterSuffix)->Increment(); - - // Feed back the recurrent state. - for (const auto& tag_pair : recurrent_fetch_tags_to_feed_tags_) { - int pos = std::find(output_name_in_signature.begin(), - output_name_in_signature.end(), tag_pair.first) - - output_name_in_signature.begin(); - inference_state->input_tensor_batches_[tag_pair.second].emplace_back( - outputs[pos]); - } - - absl::WriterMutexLock l(&mutex_); - // Set that we want to split on each index of the 0th dimension. - std::vector split_vector(options_.batch_size(), 1); - for (int i = 0; i < output_tensor_names.size(); ++i) { - if (options_.batch_size() == 1) { - if (cc->Outputs().HasTag(output_name_in_signature[i])) { - tf::Tensor output_tensor(outputs[i]); - RET_CHECK_OK(RemoveBatchDimension(&output_tensor)); - cc->Outputs() - .Tag(output_name_in_signature[i]) - .Add(new tf::Tensor(output_tensor), - inference_state->batch_timestamps_[0]); - } - } else { - std::vector split_tensors; - const tf::Status split_status = - tf::tensor::Split(outputs[i], split_vector, &split_tensors); - CHECK(split_status.ok()) << split_status.ToString(); - // Loop over timestamps so that we don't copy the padding. - for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) { - tf::Tensor output_tensor(split_tensors[j]); - RET_CHECK_OK(RemoveBatchDimension(&output_tensor)); - cc->Outputs() - .Tag(output_name_in_signature[i]) - .Add(new tf::Tensor(output_tensor), - inference_state->batch_timestamps_[j]); - } - } - } - - // Get end time and report. - const int64 end_time = absl::ToUnixMicros(clock_->TimeNow()); - cc->GetCounter(kTotalUsecsCounterSuffix) - ->IncrementBy(end_time - start_time); - cc->GetCounter(kTotalProcessedTimestampsCounterSuffix) - ->IncrementBy(inference_state->batch_timestamps_.size()); - - // Make sure we hold on to the recursive state. - if (!options_.recurrent_tag_pair().empty()) { - inference_state_ = std::move(inference_state); - inference_state_->batch_timestamps_.clear(); - } - - return absl::OkStatus(); - } - - private: - // The Session object is provided by a packet factory and is owned by the - // MediaPipe framework. Individual calls are thread-safe, but session state - // may be shared across threads. - tf::Session* session_; - - // A mapping between stream tags and the tensor names they are bound to. - std::map tag_to_tensor_map_; - - absl::Mutex mutex_; - std::unique_ptr inference_state_ ABSL_GUARDED_BY(mutex_); - - // The options for the calculator. - TensorFlowInferenceCalculatorOptions options_; - - // Store the feed and fetch tags for feed/fetch recurrent networks. - std::set recurrent_feed_tags_; - std::map recurrent_fetch_tags_to_feed_tags_; - - // Clock used to measure the computation time in OutputBatch(). - std::unique_ptr clock_; - - // The static singleton semaphore to throttle concurrent session runs. - static SimpleSemaphore* get_session_run_throttle( - int32 max_concurrent_session_runs) { - static SimpleSemaphore* session_run_throttle = - new SimpleSemaphore(max_concurrent_session_runs); - return session_run_throttle; - } -}; -REGISTER_CALCULATOR(TensorFlowInferenceCalculator); - -constexpr char TensorFlowInferenceCalculator::kTotalUsecsCounterSuffix[]; -constexpr char - TensorFlowInferenceCalculator::kTotalProcessedTimestampsCounterSuffix[]; -constexpr char - TensorFlowInferenceCalculator::kTotalSessionRunsTimeUsecsCounterSuffix[]; -constexpr char - TensorFlowInferenceCalculator::kTotalNumSessionRunsCounterSuffix[]; -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto deleted file mode 100644 index 98dbd5b4b..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TensorFlowInferenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorFlowInferenceCalculatorOptions ext = 113766539; - } - - // The signature_name specifies the mapping between stream tags and TensorFlow - // session tensors. The mapping between tags and tensors is encoded as a - // ModelManifest signature. The signatures are keyed by name and the - // named_signature matching signature_name is used by the calculator to - // match stream tags to tensors. The named_signature must be a - // ModelManifest.generic_signature with map keys that are valid tags (i.e. - // [A-Z0-9]*). - optional string signature_name = 1; - - // How many elements to batch together and feed into the graph. - // Setting the batch_size to 1 disables batching entirely. You still may or - // may not need to add the batch dimension via the option below depending on - // the input data shape and the model's expectations. - optional int32 batch_size = 2; - - // Whether to add a 0th dimension to the input tensors for batching. - // If the 0th dimension is the batch dimension, then the tensors are - // concatenated on that dimension. If the 0th is a data dimension, then a 0th - // dimension is added before concatenating. If added, the extra dimension is - // removed before outputing the tensor. Examples of each case: If you want - // to batch spectra of audio over time for an LSTM, a time-frequency - // representation has a 0th dimension as the batch dimension. If you want to - // batch frames of video that are [width, height, channels], the batch - // dimension needs to be added. - optional bool add_batch_dim_to_tensors = 3 [default = true]; - - // These pairs represent feed and fetch tensors for handling recurrent state. - // Each entry is a colon separated pair of strings. The first half of each - // string is the signature tag for the feed tensor for recurrent state. The - // second half of the string is the signature tag for the fetch tensor for the - // recurrent state. More than two colon separated strings is an error. During - // inference, The fetch tensor is fetched at every timestep and will be output - // if there is a corresponding output stream. The behavior of the feed tensor - // is determined by the following conditions in order: If the MediaPipe input - // stream with the matching tag has a packet available, then the input - // packet's tensor is passed in. If no input packet is available and we have - // fetched a tensor from the previous time step, we will feed the tensor from - // the previous timestep back in. If neither tensor is available, no tensor - // will be fed into the model. - // If this flag is set, batch_size must be 1. Do not list recurrent_tag_pair - // tags as initial_state_tags because those are only fed once. - repeated string recurrent_tag_pair = 4; - - // If set to true, skips input for which any of the features are missing. - // If set to false, requires that all input features to be available. If not, - // it will report an error for the calculator. - optional bool skip_on_missing_features = 5 [default = false]; - - // Maximum allowed concurrent Tensorflow session run calls in the calculator - // to avoid overloading local compute hardware such as TPU. Note that this - // only works in the local process, not "globally" across multiple processes - // or replicas (if any). Default to 0, i.e. no limit. - optional int32 max_concurrent_session_runs = 6 [default = 0]; - - // If turned on, the Calculator expects a vector of batched packages as input. - // This will make sure that you can turn on max_in_flight for batch_size - // greater than 1. Otherwise it results in problems of none-monotonically - // increasing timestamps. - // Use BatchSequentialCalculator to create the batches. The batch_size - // should agree for both calculators. All the data in a batch is processed - // together. The BatchSequentialCalculator can't run with max_in_flight. - optional bool batched_input = 7; -} diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc deleted file mode 100644 index 6a931679d..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc +++ /dev/null @@ -1,742 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/flags/flag.h" -#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/framework/types.h" - -#ifdef __APPLE__ -#include -#endif // defined(__APPLE__) - -namespace mediapipe { - -namespace tf = ::tensorflow; - -namespace { -std::string GetGraphDefPath() { -#ifdef __APPLE__ - char path[1024]; - CFURLRef bundle_url = CFBundleCopyBundleURL(CFBundleGetMainBundle()); - CFURLGetFileSystemRepresentation( - bundle_url, true, reinterpret_cast(path), sizeof(path)); - CFRelease(bundle_url); - return mediapipe::file::JoinPath(path, "testdata/frozen_graph_def.pb"); -#elif defined(__ANDROID__) - char path[1024]; - getcwd(path, sizeof(path)); - return mediapipe::file::JoinPath(path, - "mediapipe/calculators/tensorflow/" - "testdata/frozen_graph_def.pb"); -#else - return mediapipe::file::JoinPath( - "./", - // This should match the path of the output files - // of the genrule() that generates test model files. - "mediapipe/calculators/tensorflow/testdata/", "frozen_graph_def.pb"); -#endif // defined(__APPLE__) -} -} // namespace - -class TensorflowInferenceCalculatorTest : public ::testing::Test { - protected: - // Add the input side packet. - void AddSessionInputSidePacket() { - PacketGeneratorOptions extendable_options; - TensorFlowSessionFromFrozenGraphGeneratorOptions* generator_options; - generator_options = extendable_options.MutableExtension( - TensorFlowSessionFromFrozenGraphGeneratorOptions::ext); - generator_options->set_graph_proto_path(GetGraphDefPath()); - (*generator_options->mutable_tag_to_tensor_names())["MULTIPLIED"] = - "multiplied:0"; - (*generator_options->mutable_tag_to_tensor_names())["A"] = "a:0"; - (*generator_options->mutable_tag_to_tensor_names())["B"] = "b:0"; - (*generator_options->mutable_tag_to_tensor_names())["EXPENSIVE"] = - "expensive:0"; - - PacketSet input_side_packets({}); - PacketSet output_side_packets({"SESSION"}); - MEDIAPIPE_CHECK_OK(tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options, - input_side_packets, &output_side_packets)); - runner_->MutableSidePackets()->Tag("SESSION") = - output_side_packets.Tag("SESSION"); - } - - Packet CreateTensorPacket(const std::vector& input, int64 time) { - tf::TensorShape tensor_shape; - tensor_shape.AddDim(input.size()); - auto tensor = absl::make_unique(tf::DT_INT32, tensor_shape); - for (int i = 0; i < input.size(); ++i) { - tensor->vec()(i) = input[i]; - } - return Adopt(tensor.release()).At(Timestamp(time)); - } - - // Create tensor from Vector and add as a Packet to the provided tag as input. - void AddVectorToInputsAsTensor(const std::vector& input, - const std::string& tag, int64 time) { - runner_->MutableInputs()->Tag(tag).packets.push_back( - CreateTensorPacket(input, time)); - } - - // Create tensor from Vector and add as a Packet to the provided tag as input. - void AddVectorToInputsAsPacket(const std::vector& packets, - const std::string& tag) { - CHECK(!packets.empty()) - << "Please specify at least some data in the packet"; - auto packets_ptr = absl::make_unique>(packets); - runner_->MutableInputs()->Tag(tag).packets.push_back( - Adopt(packets_ptr.release()).At(packets.begin()->Timestamp())); - } - - std::unique_ptr runner_; -}; - -TEST_F(TensorflowInferenceCalculatorTest, GetConstants) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_in"); - config.add_output_stream("B:tensor_out"); - config.add_output_stream("MULTIPLIED:tensor_multiplied"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(false); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({0, 0, 0}, "A", 0); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_b = - runner_->Outputs().Tag("B").packets; - ASSERT_EQ(output_packets_b.size(), 1); - const tf::Tensor& tensor_b = output_packets_b[0].Get(); - tf::TensorShape expected_shape({1, 3}); - auto expected_tensor = tf::test::AsTensor({3, 2, 1}, expected_shape); - tf::test::ExpectTensorEqual(expected_tensor, tensor_b); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(1, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - expected_tensor = tf::test::AsTensor({0, 0, 0}, expected_shape); - tf::test::ExpectTensorEqual(expected_tensor, tensor_mult); - - EXPECT_EQ(1, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetComputed) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(false); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(1, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - tf::TensorShape expected_shape({3}); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}, expected_shape); - tf::test::ExpectTensorEqual(expected_tensor, tensor_mult); - - // Add only one of the two expected tensors at the next timestamp, expect - // useful failure message. - AddVectorToInputsAsTensor({1, 2, 3}, "A", 1); - auto run_status = runner_->Run(); - ASSERT_FALSE(run_status.ok()); - EXPECT_THAT(run_status.ToString(), - testing::HasSubstr("TensorFlowInferenceCalculator")); - EXPECT_THAT(run_status.ToString(), testing::HasSubstr("Tag B")); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetComputed_MaxInFlight) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - config.set_max_in_flight(2); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(false); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(1, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - tf::TensorShape expected_shape({3}); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}, expected_shape); - tf::test::ExpectTensorEqual(expected_tensor, tensor_mult); - - // Add only one of the two expected tensors at the next timestamp, expect - // useful failure message. - AddVectorToInputsAsTensor({1, 2, 3}, "A", 1); - auto run_status = runner_->Run(); - ASSERT_FALSE(run_status.ok()); - EXPECT_THAT(run_status.ToString(), - testing::HasSubstr("TensorFlowInferenceCalculator")); - EXPECT_THAT(run_status.ToString(), testing::HasSubstr("Tag B")); -} - -TEST_F(TensorflowInferenceCalculatorTest, BadTag) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("BAD:tensor_in"); // This one is bad. - config.add_output_stream("B:tensor_out"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - ASSERT_FALSE(runner_->Run().ok()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetMultiBatchComputed_MaxInFlight) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - config.set_max_in_flight(2); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, - GetMultiBatchComputed_MoreThanMaxInFlight) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - config.set_max_in_flight(2); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - AddVectorToInputsAsTensor({4, 4, 4}, "A", 2); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 2); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(3, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - const tf::Tensor& tensor_mult2 = output_packets_mult[2].Get(); - auto expected_tensor2 = tf::test::AsTensor({12, 16, 20}); - tf::test::ExpectTensorEqual(tensor_mult2, expected_tensor2); - - EXPECT_EQ(3, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetSingleBatchComputed) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetCloseBatchComputed) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(3); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, GetBatchComputed_MaxInFlight) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - config.set_max_in_flight(2); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batched_input(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsPacket( - {CreateTensorPacket({2, 2, 2}, 0), CreateTensorPacket({3, 3, 3}, 1)}, - "A"); - AddVectorToInputsAsPacket( - {CreateTensorPacket({3, 4, 5}, 0), CreateTensorPacket({3, 4, 5}, 1)}, - "B"); - AddVectorToInputsAsPacket( - {CreateTensorPacket({4, 4, 4}, 2), CreateTensorPacket({5, 5, 5}, 3)}, - "A"); - AddVectorToInputsAsPacket( - {CreateTensorPacket({3, 4, 5}, 2), CreateTensorPacket({3, 4, 5}, 3)}, - "B"); - AddVectorToInputsAsPacket({CreateTensorPacket({6, 6, 6}, 4)}, "A"); - AddVectorToInputsAsPacket({CreateTensorPacket({3, 4, 5}, 4)}, "B"); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(5, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({6, 8, 10}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - const tf::Tensor& tensor_mult2 = output_packets_mult[2].Get(); - auto expected_tensor2 = tf::test::AsTensor({12, 16, 20}); - tf::test::ExpectTensorEqual(tensor_mult2, expected_tensor2); - const tf::Tensor& tensor_mult3 = output_packets_mult[3].Get(); - auto expected_tensor3 = tf::test::AsTensor({15, 20, 25}); - tf::test::ExpectTensorEqual(tensor_mult3, expected_tensor3); - const tf::Tensor& tensor_mult4 = output_packets_mult[4].Get(); - auto expected_tensor4 = tf::test::AsTensor({18, 24, 30}); - tf::test::ExpectTensorEqual(tensor_mult4, expected_tensor4); - - EXPECT_EQ(5, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->add_recurrent_tag_pair("A:MULTIPLIED"); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - LOG(INFO) << "timestamp: " << 0; - auto expected_tensor = tf::test::AsTensor({3, 8, 15}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({9, 32, 75}); - LOG(INFO) << "timestamp: " << 1; - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} -TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->add_recurrent_tag_pair("A:MULTIPLIED"); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({1, 1, 1}, "A", 0); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 0); - AddVectorToInputsAsTensor({1, 1, 1}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(2, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - LOG(INFO) << "timestamp: " << 0; - auto expected_tensor = tf::test::AsTensor({3, 4, 5}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); - auto expected_tensor1 = tf::test::AsTensor({3, 4, 5}); - LOG(INFO) << "timestamp: " << 1; - tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); - - EXPECT_EQ(2, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -// TODO: Investigate this test failure. -TEST_F(TensorflowInferenceCalculatorTest, DISABLED_CheckTiming) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_in"); - config.add_output_stream("EXPENSIVE:tensor_expensive"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(1); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(false); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({0, 0, 0}, "A", 0); - MP_ASSERT_OK(runner_->Run()); - - EXPECT_EQ(1, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); - // We only test the timing counter here because we are requesting an - // expensive tensor output. Because the precision on android is - // sometimes closer to milliseconds, we need to request a large tensor - // to be sure this will be greater than zero. - EXPECT_GT(runner_->GetCounter("TensorFlowInferenceCalculator-TotalTimeUsecs") - ->Get(), - 0); -} - -TEST_F(TensorflowInferenceCalculatorTest, MissingInputFeature) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_skip_on_missing_features(false); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - ASSERT_FALSE(runner_->Run().ok()); -} - -TEST_F(TensorflowInferenceCalculatorTest, MissingInputFeature_Skip) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_skip_on_missing_features(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(0, output_packets_mult.size()); -} - -TEST_F(TensorflowInferenceCalculatorTest, - MissingInputFeature_SkipCheckInternalState) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_skip_on_missing_features(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsTensor({2, 2, 2}, "A", 0); - AddVectorToInputsAsTensor({3, 3, 3}, "A", 1); - AddVectorToInputsAsTensor({3, 4, 5}, "B", 1); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets_mult = - runner_->Outputs().Tag("MULTIPLIED").packets; - ASSERT_EQ(1, output_packets_mult.size()); - const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - auto expected_tensor = tf::test::AsTensor({9, 12, 15}); - tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); - - EXPECT_EQ(1, runner_ - ->GetCounter( - "TensorFlowInferenceCalculator-TotalProcessedTimestamps") - ->Get()); -} - -TEST_F(TensorflowInferenceCalculatorTest, BatchedInputTooBigBatch) { - CalculatorGraphConfig::Node config; - config.set_calculator("TensorFlowInferenceCalculator"); - config.add_input_stream("A:tensor_a"); - config.add_input_stream("B:tensor_b"); - config.add_output_stream("MULTIPLIED:tensor_o1"); - config.add_input_side_packet("SESSION:session"); - config.set_max_in_flight(2); - CalculatorOptions options; - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batch_size(2); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_add_batch_dim_to_tensors(true); - options.MutableExtension(TensorFlowInferenceCalculatorOptions::ext) - ->set_batched_input(true); - *config.mutable_options() = options; - - runner_ = absl::make_unique(config); - AddSessionInputSidePacket(); - AddVectorToInputsAsPacket( - {CreateTensorPacket({2, 2, 2}, 0), CreateTensorPacket({3, 3, 3}, 1), - CreateTensorPacket({4, 4, 4}, 2)}, - "A"); - AddVectorToInputsAsPacket( - {CreateTensorPacket({3, 4, 5}, 0), CreateTensorPacket({3, 4, 5}, 1), - CreateTensorPacket({3, 4, 5}, 2)}, - "B"); - - auto status = runner_->Run(); - ASSERT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - ::testing::HasSubstr( - "has more packets than batch capacity. batch_size: 2 packets: 3")); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session.h b/mediapipe/calculators/tensorflow/tensorflow_session.h deleted file mode 100644 index 250e2a572..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_TENSORFLOW_CALCULATORS_TENSORFLOW_SESSION_H_ -#define MEDIAPIPE_TENSORFLOW_CALCULATORS_TENSORFLOW_SESSION_H_ - -#include - -#include "tensorflow/core/public/session.h" - -namespace mediapipe { -struct TensorFlowSession { - // TensorFlow session wrapper to get around the RTTI issue. - std::unique_ptr session; - - // Store an optional mapping to the between MediaPipe tags and TensorFlow - // tensor names. Creating this mapping when the session is loaded allows more - // flexible definition of mapping tags to tensors across platforms. - std::map tag_to_tensor_map; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_TENSORFLOW_CALCULATORS_TENSORFLOW_SESSION_H_ diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc deleted file mode 100644 index 2c1d169bc..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Reads serialized GraphDef proto. There are three ways to load a model: -// 1. Specify the path to a graph.pb in the calculator options. -// 2. Specify the path to the graph.pb through the -// input_side_packet:STRING_MODEL_FILE_PATH -// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL, -// typically provided by EmbeddingFilePacketFactory. -// -// Produces a SessionBundle that TensorFlowInferenceCalculator can use. - -#include - -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/status_util.h" -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/public/session_options.h" - -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -namespace tf = ::tensorflow; - -namespace { -// Updates the graph nodes to use the device as specified by device_id. -void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) { - for (auto& node : *graph_def->mutable_node()) { - if (node.device().empty()) { - node.set_device(std::string(device_id)); - } - } -} -} // namespace - -class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = - cc->Options(); - bool has_exactly_one_model = - !options.graph_proto_path().empty() - ? !(cc->InputSidePackets().HasTag("STRING_MODEL") | - cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) - : (cc->InputSidePackets().HasTag("STRING_MODEL") ^ - cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")); - RET_CHECK(has_exactly_one_model) - << "Must have exactly one of graph_proto_path in options or " - "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH"; - if (cc->InputSidePackets().HasTag("STRING_MODEL")) { - cc->InputSidePackets() - .Tag("STRING_MODEL") - .Set( - // String model from embedded path - ); - } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) { - cc->InputSidePackets() - .Tag("STRING_MODEL_FILE_PATH") - .Set( - // Filename of std::string model. - ); - } - cc->OutputSidePackets().Tag("SESSION").Set( - // A TensorFlow model loaded and ready for use along with - // a map from tags to tensor names. - ); - RET_CHECK_GT(options.tag_to_tensor_names().size(), 0); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - auto clock = std::unique_ptr( - mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock()); - const uint64 start_time = absl::ToUnixMicros(clock->TimeNow()); - const auto& options = - cc->Options(); - // Output bundle packet. - auto session = ::absl::make_unique(); - - tf::SessionOptions session_options; - session_options.config.CopyFrom(options.config()); - std::vector initialization_op_names; - initialization_op_names.reserve(options.initialization_op_names_size()); - for (int i = 0; i < options.initialization_op_names_size(); ++i) { - initialization_op_names.emplace_back(options.initialization_op_names(i)); - } - session->session.reset(tf::NewSession(session_options)); - - std::string graph_def_serialized; - if (cc->InputSidePackets().HasTag("STRING_MODEL")) { - graph_def_serialized = - cc->InputSidePackets().Tag("STRING_MODEL").Get(); - } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) { - const std::string& frozen_graph = cc->InputSidePackets() - .Tag("STRING_MODEL_FILE_PATH") - .Get(); - RET_CHECK_OK( - mediapipe::file::GetContents(frozen_graph, &graph_def_serialized)); - } else { - RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(), - &graph_def_serialized)); - } - tensorflow::GraphDef graph_def; - - RET_CHECK(graph_def.ParseFromString(graph_def_serialized)); - - // Update the graph nodes to use the preferred device, if set. - if (!options.preferred_device_id().empty()) { - SetPreferredDevice(&graph_def, options.preferred_device_id()); - } - - const tf::Status tf_status = session->session->Create(graph_def); - RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString(); - - for (const auto& key_value : options.tag_to_tensor_names()) { - session->tag_to_tensor_map[key_value.first] = key_value.second; - } - if (!initialization_op_names.empty()) { - const tf::Status tf_status = - session->session->Run({}, {}, initialization_op_names, {}); - // RET_CHECK on the tf::Status object itself in order to print an - // informative error message. - RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString(); - } - - cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release())); - const uint64 end_time = absl::ToUnixMicros(clock->TimeNow()); - LOG(INFO) << "Loaded frozen model in: " << end_time - start_time - << " microseconds."; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(TensorFlowSessionFromFrozenGraphCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto deleted file mode 100644 index 87b2304ad..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "tensorflow/core/protobuf/config.proto"; - -message TensorFlowSessionFromFrozenGraphCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorFlowSessionFromFrozenGraphCalculatorOptions ext = 266997877; - } - - // Path to file containing serialized proto of type tensorflow::GraphDef. - optional string graph_proto_path = 1; - - // To run inference with MediaPipe inputs MediaPipe streams need to be mapped - // to TensorFlow tensors. This map defines the which streams are fed into - // which tensors in the model. The MediaPipe tag of the stream is the map key. - // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING" - // and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of - // input_stream or output_stream of the TensorflowInferenceCalculator - // consuming the packet produced by this calculator. The tensor names must - // match the tensor names in the graph that you want to feed or fetch into or - // out of. Examples: "DecodeJpeg/contents:0" or "softmax:0". For example, a - // mediapipe graph can include the nodes: - // - // node { - // calculator: "TensorFlowSessionFromFrozenGraphCalculator" - // output_side_packet: "SESSION:session" - // options { - // [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - // graph_proto_path: "[PATH]" - // tag_to_tensor_names { - // key: "JPG_STRING" - // value: "input:0" - // } - // tag_to_tensor_names { - // key: "SOFTMAX" - // value: "softmax:0" - // } - // } - // } - // } - // node { - // calculator: "TensorflowInferenceCalculator" - // input_side_packet: "SESSION:graph_with_bindings" - // input_stream: "JPG_STRING:jpg_string_tensor" - // output_stream: "SOFTMAX:softmax_tensor" - // } - map tag_to_tensor_names = 2; - - // Tensorflow session config options. - optional tensorflow.ConfigProto config = 3; - - // Graph nodes to run to initialize the model. Any output of these ops is - // ignored. - repeated string initialization_op_names = 4; - - // The id of the device you would prefer to execute the graph nodes on. - // If set, all graph nodes without a previously specified device, will be set - // to run on preferred_device_id. Example values include: - // ["/device:GPU:0","/device:CPU:0", ...] - // NOTE: If config.allow_soft_placement = false, and the device is not found, - // an error will be thrown. - optional string preferred_device_id = 5; -} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc deleted file mode 100644 index bdf90dcbb..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "absl/strings/substitute.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/tag_map_helper.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/protobuf/config.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -std::string GetGraphDefPath() { - return mediapipe::file::JoinPath("./", - "mediapipe/calculators/tensorflow/" - "testdata/frozen_graph_def.pb"); -} - -// Helper function that creates Tensor INT32 matrix with size 1x3. -tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) { - tf::Tensor tensor(tf::DT_INT32, - tf::TensorShape(std::vector({1, 3}))); - auto matrix = tensor.matrix(); - matrix(0, 0) = v1; - matrix(0, 1) = v2; - matrix(0, 2) = v3; - return tensor; -} - -class TensorFlowSessionFromFrozenGraphCalculatorTest : public ::testing::Test { - protected: - void SetUp() override { - extendable_options_.Clear(); - calculator_options_ = extendable_options_.MutableExtension( - TensorFlowSessionFromFrozenGraphCalculatorOptions::ext); - calculator_options_->set_graph_proto_path(GetGraphDefPath()); - (*calculator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] = - "multiplied:0"; - (*calculator_options_->mutable_tag_to_tensor_names())["A"] = "a:0"; - (*calculator_options_->mutable_tag_to_tensor_names())["B"] = "b:0"; - calculator_options_->mutable_config()->set_intra_op_parallelism_threads(1); - calculator_options_->mutable_config()->set_inter_op_parallelism_threads(2); - calculator_options_->set_preferred_device_id("/device:CPU:0"); - } - - void VerifySignatureMap(const TensorFlowSession& session) { - // Session must be set. - ASSERT_NE(session.session, nullptr); - - // Bindings are inserted. - EXPECT_EQ(session.tag_to_tensor_map.size(), 3); - - // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators. - EXPECT_FALSE(session.tag_to_tensor_map.find("A") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("B") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") == - session.tag_to_tensor_map.end()); - // Sanity: find() actually returns a reference to end() if element not - // found. - EXPECT_TRUE(session.tag_to_tensor_map.find("Z") == - session.tag_to_tensor_map.end()); - - EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0"); - } - - CalculatorOptions extendable_options_; - TensorFlowSessionFromFrozenGraphCalculatorOptions* calculator_options_; -}; - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CreatesPacketWithGraphAndBindings) { - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - - MP_ASSERT_OK(runner.Run()); - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - VerifySignatureMap(session); -} - -// Integration test. Verifies that TensorFlowInferenceCalculator correctly -// consumes the Packet emitted by this calculator. -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - ProducesPacketUsableByTensorFlowInferenceCalculator) { - CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - node { - calculator: "TensorFlowInferenceCalculator" - input_side_packet: "SESSION:session" - input_stream: "A:a_tensor" - output_stream: "MULTIPLIED:multiplied_tensor" - options { - [mediapipe.TensorFlowInferenceCalculatorOptions.ext] { - batch_size: 5 - add_batch_dim_to_tensors: false - } - } - } - - node { - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - } - } - input_stream: "a_tensor" - )", - calculator_options_->DebugString())); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("multiplied_tensor"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "a_tensor", - Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("a_tensor")); - - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - // input tensor gets multiplied by [[3, 2, 1]]. Expected output: - tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10); - EXPECT_EQ(expected_multiplication.DebugString(), - packet.Get().DebugString()); - - ASSERT_FALSE(poller.Next(&packet)); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CreatesPacketWithGraphAndBindingsFromInputSidePacket) { - calculator_options_->clear_graph_proto_path(); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - input_side_packet: "STRING_MODEL:model" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - runner.MutableSidePackets()->Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - MP_ASSERT_OK(runner.Run()); - - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - VerifySignatureMap(session); -} - -TEST_F( - TensorFlowSessionFromFrozenGraphCalculatorTest, - CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) { - calculator_options_->clear_graph_proto_path(); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - input_side_packet: "STRING_MODEL_FILE_PATH:file_path" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - MP_ASSERT_OK(runner.Run()); - - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - VerifySignatureMap(session); -} - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CheckFailureForOptionsAndInputsProvideGraphDefProto) { - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - input_side_packet: "STRING_MODEL_FILE_PATH:file_path" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - auto run_status = runner.Run(); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CheckFailureForAllInputsProvideGraphDefProto) { - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - input_side_packet: "STRING_MODEL_FILE_PATH:file_path" - input_side_packet: "STRING_MODEL:model" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - runner.MutableSidePackets()->Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - auto run_status = runner.Run(); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) { - calculator_options_->clear_graph_proto_path(); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - input_side_packet: "STRING_MODEL_FILE_PATH:file_path" - input_side_packet: "STRING_MODEL:model" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - runner.MutableSidePackets()->Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - auto run_status = runner.Run(); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, - CheckInitializationOpName) { - calculator_options_->add_initialization_op_names("multiplied:0"); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromFrozenGraphCalculator" - output_side_packet: "SESSION:session" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { - $0 - } - })", - calculator_options_->DebugString())); - MP_ASSERT_OK(runner.Run()); - - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - VerifySignatureMap(session); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc deleted file mode 100644 index 9f5b9e06b..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Reads serialized GraphDef proto. There are three ways to load a model: -// 1. Specify the path to a graph.pb in the calculator options. -// 2. Specify the path to the graph.pb through the -// input_side_packet:STRING_MODEL_FILE_PATH -// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL, -// typically provided by EmbeddingFilePacketFactory. -// -// See tensorflow_session_bundle_from_graph_generator.proto for options. -// Produces a SessionBundle that TensorFlowInferenceCalculator can use. - -#include - -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/status_util.h" -#include "tensorflow/core/framework/graph.pb.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/public/session_options.h" - -namespace mediapipe { - -namespace tf = ::tensorflow; - -namespace { -// Updates the graph nodes to use the device as specified by device_id. -void SetPreferredDevice(tf::GraphDef* graph_def, absl::string_view device_id) { - for (auto& node : *graph_def->mutable_node()) { - if (node.device().empty()) { - node.set_device(std::string(device_id)); - } - } -} -} // namespace - -class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator { - public: - static absl::Status FillExpectations( - const PacketGeneratorOptions& extendable_options, - PacketTypeSet* input_side_packets, PacketTypeSet* output_side_packets) { - RET_CHECK(extendable_options.HasExtension( - TensorFlowSessionFromFrozenGraphGeneratorOptions::ext)); - const auto& options = extendable_options.GetExtension( // NOLINT - TensorFlowSessionFromFrozenGraphGeneratorOptions::ext); - bool has_exactly_one_model = - !options.graph_proto_path().empty() - ? !(input_side_packets->HasTag("STRING_MODEL") | - input_side_packets->HasTag("STRING_MODEL_FILE_PATH")) - : (input_side_packets->HasTag("STRING_MODEL") ^ - input_side_packets->HasTag("STRING_MODEL_FILE_PATH")); - RET_CHECK(has_exactly_one_model) - << "Must have exactly one of graph_proto_path in options or " - "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH"; - if (input_side_packets->HasTag("STRING_MODEL")) { - input_side_packets->Tag("STRING_MODEL") - .Set( - // String model from embedded path - ); - } else if (input_side_packets->HasTag("STRING_MODEL_FILE_PATH")) { - input_side_packets->Tag("STRING_MODEL_FILE_PATH") - .Set( - // Filename of std::string model. - ); - } - output_side_packets->Tag("SESSION").Set( - // A TensorFlow model loaded and ready for use along with - // a map from tags to tensor names. - ); - RET_CHECK_GT(options.tag_to_tensor_names().size(), 0); - return absl::OkStatus(); - } - - static absl::Status Generate( - const PacketGeneratorOptions& packet_generator_options, - const PacketSet& input_side_packets, PacketSet* output_side_packets) { - auto clock = std::unique_ptr( - mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock()); - const uint64 start_time = absl::ToUnixMicros(clock->TimeNow()); - const TensorFlowSessionFromFrozenGraphGeneratorOptions& options = - packet_generator_options.GetExtension( - TensorFlowSessionFromFrozenGraphGeneratorOptions::ext); - // Output bundle packet. - auto session = ::absl::make_unique(); - - tf::SessionOptions session_options; - session_options.config.CopyFrom(options.config()); - std::vector initialization_op_names; - initialization_op_names.reserve(options.initialization_op_names_size()); - for (int i = 0; i < options.initialization_op_names_size(); ++i) { - initialization_op_names.emplace_back(options.initialization_op_names(i)); - } - session->session.reset(tf::NewSession(session_options)); - - std::string graph_def_serialized; - if (input_side_packets.HasTag("STRING_MODEL")) { - graph_def_serialized = - input_side_packets.Tag("STRING_MODEL").Get(); - } else if (input_side_packets.HasTag("STRING_MODEL_FILE_PATH")) { - const std::string& frozen_graph = - input_side_packets.Tag("STRING_MODEL_FILE_PATH").Get(); - RET_CHECK_OK( - mediapipe::file::GetContents(frozen_graph, &graph_def_serialized)); - } else { - RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(), - &graph_def_serialized)); - } - tensorflow::GraphDef graph_def; - - RET_CHECK(graph_def.ParseFromString(graph_def_serialized)); - - // Update the graph nodes to use the preferred device, if set. - if (!options.preferred_device_id().empty()) { - SetPreferredDevice(&graph_def, options.preferred_device_id()); - } - - const tf::Status tf_status = session->session->Create(graph_def); - RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.ToString(); - - for (const auto& key_value : options.tag_to_tensor_names()) { - session->tag_to_tensor_map[key_value.first] = key_value.second; - } - if (!initialization_op_names.empty()) { - const tf::Status tf_status = - session->session->Run({}, {}, initialization_op_names, {}); - // RET_CHECK on the tf::Status object itself in order to print an - // informative error message. - RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.ToString(); - } - - output_side_packets->Tag("SESSION") = Adopt(session.release()); - const uint64 end_time = absl::ToUnixMicros(clock->TimeNow()); - LOG(INFO) << "Loaded frozen model in: " << end_time - start_time - << " microseconds."; - return absl::OkStatus(); - } -}; -REGISTER_PACKET_GENERATOR(TensorFlowSessionFromFrozenGraphGenerator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.proto deleted file mode 100644 index 4643b4d60..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.proto +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/packet_generator.proto"; -import "tensorflow/core/protobuf/config.proto"; - -message TensorFlowSessionFromFrozenGraphGeneratorOptions { - extend mediapipe.PacketGeneratorOptions { - optional TensorFlowSessionFromFrozenGraphGeneratorOptions ext = 160666123; - } - - // Path to file containing serialized proto of type tensorflow::GraphDef. - optional string graph_proto_path = 1; - - // To run inference with MediaPipe inputs MediaPipe streams need to be mapped - // to TensorFlow tensors. This map defines the which streams are fed into - // which tensors in the model. The MediaPipe tag of the stream is the map key. - // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING" - // and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of - // input_stream or output_stream of the TensorflowInferenceCalculator - // consuming the packet produced by this generator. The tensor names must - // match the tensor names in the graph that you want to feed or fetch into or - // out of. Examples: "DecodeJpeg/contents:0" or "softmax:0". For example, a - // mediapipe graph can include the nodes: - // - // packet_generator { - // packet_generator: "TensorFlowSessionFromFrozenGraphGenerator" - // output_side_packet: "SESSION:session" - // options { - // [mediapipe.TensorFlowSessionFromFrozenGraphGeneratorOptions.ext]: { - // graph_proto_path: "[PATH]" - // tag_to_tensor_names { - // key: "JPG_STRING" - // value: "input:0" - // } - // tag_to_tensor_names { - // key: "SOFTMAX" - // value: "softmax:0" - // } - // } - // } - // } - // node { - // calculator: "TensorflowInferenceCalculator" - // input_side_packet: "SESSION:graph_with_bindings" - // input_stream: "JPG_STRING:jpg_string_tensor" - // output_stream: "SOFTMAX:softmax_tensor" - // } - map tag_to_tensor_names = 2; - - // Tensorflow session config options. - optional tensorflow.ConfigProto config = 3; - - // Graph nodes to run to initialize the model. Any output of these ops is - // ignored. - repeated string initialization_op_names = 4; - - // The id of the device you would prefer to execute the graph nodes on. - // If set, all graph nodes without a previously specified device, will be set - // to run on preferred_device_id. Example values include: - // ["/device:GPU:0","/device:CPU:0", ...] - // NOTE: If config.allow_soft_placement = false, and the device is not found, - // an error will be thrown. - optional string preferred_device_id = 5; -} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc deleted file mode 100644 index 34d7e8828..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator_test.cc +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "absl/strings/substitute.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/packet_generator.pb.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/tag_map_helper.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/protobuf/config.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -std::string GetGraphDefPath() { - return mediapipe::file::JoinPath("./", - "mediapipe/calculators/tensorflow/" - "testdata/frozen_graph_def.pb"); -} - -// Helper function that creates Tensor INT32 matrix with size 1x3. -tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) { - tf::Tensor tensor(tf::DT_INT32, - tf::TensorShape(std::vector({1, 3}))); - auto matrix = tensor.matrix(); - matrix(0, 0) = v1; - matrix(0, 1) = v2; - matrix(0, 2) = v3; - return tensor; -} - -class TensorFlowSessionFromFrozenGraphGeneratorTest : public ::testing::Test { - protected: - void SetUp() override { - extendable_options_.Clear(); - generator_options_ = extendable_options_.MutableExtension( - TensorFlowSessionFromFrozenGraphGeneratorOptions::ext); - generator_options_->set_graph_proto_path(GetGraphDefPath()); - (*generator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] = - "multiplied:0"; - (*generator_options_->mutable_tag_to_tensor_names())["A"] = "a:0"; - (*generator_options_->mutable_tag_to_tensor_names())["B"] = "b:0"; - generator_options_->mutable_config()->set_intra_op_parallelism_threads(1); - generator_options_->mutable_config()->set_inter_op_parallelism_threads(2); - generator_options_->set_preferred_device_id("/device:CPU:0"); - } - - void VerifySignatureMap(PacketSet* output_side_packets) { - const TensorFlowSession& session = - output_side_packets->Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); - - // Bindings are inserted. - EXPECT_EQ(session.tag_to_tensor_map.size(), 3); - - // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators. - EXPECT_FALSE(session.tag_to_tensor_map.find("A") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("B") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") == - session.tag_to_tensor_map.end()); - // Sanity: find() actually returns a reference to end() if element not - // found. - EXPECT_TRUE(session.tag_to_tensor_map.find("Z") == - session.tag_to_tensor_map.end()); - - EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0"); - } - - PacketGeneratorOptions extendable_options_; - TensorFlowSessionFromFrozenGraphGeneratorOptions* generator_options_; -}; - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CreatesPacketWithGraphAndBindings) { - PacketSet input_side_packets(tool::CreateTagMap({}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - VerifySignatureMap(&output_side_packets); -} - -// Integration test. Verifies that TensorFlowInferenceCalculator correctly -// consumes the Packet emitted by this generator. -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - ProducesPacketUsableByTensorFlowInferenceCalculator) { - CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - node { - calculator: "TensorFlowInferenceCalculator" - input_side_packet: "SESSION:tf_model" - input_stream: "A:a_tensor" - output_stream: "MULTIPLIED:multiplied_tensor" - options { - [mediapipe.TensorFlowInferenceCalculatorOptions.ext] { - batch_size: 5 - add_batch_dim_to_tensors: false - } - } - } - - packet_generator { - packet_generator: "TensorFlowSessionFromFrozenGraphGenerator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromFrozenGraphGeneratorOptions.ext]: { - $0 - } - } - } - input_stream: "a_tensor" - )", - generator_options_->DebugString())); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("multiplied_tensor"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "a_tensor", - Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("a_tensor")); - - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - // input tensor gets multiplied by [[3, 2, 1]]. Expected output: - tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10); - EXPECT_EQ(expected_multiplication.DebugString(), - packet.Get().DebugString()); - - ASSERT_FALSE(poller.Next(&packet)); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CreatesPacketWithGraphAndBindingsFromInputSidePacket) { - PacketSet input_side_packets( - tool::CreateTagMap({"STRING_MODEL:model"}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - generator_options_->clear_graph_proto_path(); - input_side_packets.Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - VerifySignatureMap(&output_side_packets); -} - -TEST_F( - TensorFlowSessionFromFrozenGraphGeneratorTest, - CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) { - PacketSet input_side_packets( - tool::CreateTagMap({"STRING_MODEL_FILE_PATH:model_path"}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - generator_options_->clear_graph_proto_path(); - input_side_packets.Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - VerifySignatureMap(&output_side_packets); -} - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CheckFailureForOptionsAndInputsProvideGraphDefProto) { - PacketSet input_side_packets( - tool::CreateTagMap({"STRING_MODEL_FILE_PATH:model_path"}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - input_side_packets.Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - EXPECT_EQ(run_status.code(), absl::StatusCode::kInternal); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CheckFailureForAllInputsProvideGraphDefProto) { - PacketSet input_side_packets( - tool::CreateTagMap( - {"STRING_MODEL_FILE_PATH:model_path", "STRING_MODEL:model"}) - .value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - input_side_packets.Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - input_side_packets.Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - EXPECT_EQ(run_status.code(), absl::StatusCode::kInternal); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) { - PacketSet input_side_packets( - tool::CreateTagMap( - {"STRING_MODEL_FILE_PATH:model_path", "STRING_MODEL:model"}) - .value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - std::string serialized_graph_contents; - MP_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), - &serialized_graph_contents)); - input_side_packets.Tag("STRING_MODEL") = - Adopt(new std::string(serialized_graph_contents)); - input_side_packets.Tag("STRING_MODEL_FILE_PATH") = - Adopt(new std::string(GetGraphDefPath())); - generator_options_->clear_graph_proto_path(); - - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - EXPECT_EQ(run_status.code(), absl::StatusCode::kInternal); - EXPECT_THAT( - run_status.message(), - ::testing::HasSubstr("Must have exactly one of graph_proto_path")); -} - -TEST_F(TensorFlowSessionFromFrozenGraphGeneratorTest, - CheckInitializationOpName) { - PacketSet input_side_packets(tool::CreateTagMap({}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - generator_options_->add_initialization_op_names("multiplied:0"); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromFrozenGraphGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status); - VerifySignatureMap(&output_side_packets); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc deleted file mode 100644 index c169c6b1e..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#if !defined(__ANDROID__) -#include "mediapipe/framework/port/file_helpers.h" -#endif -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/cc/saved_model/constants.h" -#include "tensorflow/cc/saved_model/loader.h" -#include "tensorflow/cc/saved_model/tag_constants.h" - -namespace mediapipe { - -namespace { -static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH"; - -// Given the path to a directory containing multiple tensorflow saved models -// in subdirectories, replaces path with the alphabetically last subdirectory. -absl::Status GetLatestDirectory(std::string* path) { -#if defined(__ANDROID__) - return absl::UnimplementedError( - "GetLatestDirectory is not implemented on Android"); -#else - std::vector saved_models; - RET_CHECK_OK(file::MatchInTopSubdirectories( - *path, tensorflow::kSavedModelFilenamePb, &saved_models)); - RET_CHECK_GT(saved_models.size(), 0) - << "No exported bundles found in " << path; - ::std::sort(saved_models.begin(), saved_models.end()); - *path = std::string(file::Dirname(saved_models.back())); - return absl::OkStatus(); -#endif -} - -// If options.convert_signature_to_tags() is set, will convert letters to -// uppercase and replace /'s and -'s with _'s. This enables the standard -// SavedModel classification, regression, and prediction signatures to be used -// as uppercase INPUTS and OUTPUTS tags for streams and supports other common -// patterns. -const std::string MaybeConvertSignatureToTag( - const std::string& name, - const TensorFlowSessionFromSavedModelCalculatorOptions& options) { - if (options.convert_signature_to_tags()) { - std::string output; - output.resize(name.length()); - std::transform(name.begin(), name.end(), output.begin(), - [](unsigned char c) { return std::toupper(c); }); - output = absl::StrReplaceAll(output, {{"/", "_"}}); - output = absl::StrReplaceAll(output, {{"-", "_"}}); - return output; - } else { - return name; - } -} - -} // namespace - -// TensorFlowSessionFromSavedModelCalculator is a MediaPipe packet calculator -// that loads a trained TensorFlow model exported via SavedModel's exporter and -// returns a Packet containing a unique_ptr to a mediapipe::TensorFlowSession, -// which in turn contains a TensorFlow Session ready for execution and a map -// between tags and tensor names. -// -// -// Example usage: -// node { -// calculator: "TensorFlowSessionFromSavedModelCalculator" -// output_side_packet: "SESSION:vod_session" -// options { -// [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { -// signature_name: "serving_default" -// saved_model_path: "path/to/model" -// } -// } -// } -class TensorFlowSessionFromSavedModelCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = - cc->Options(); - const bool has_exactly_one_model = - options.saved_model_path().empty() == - cc->InputSidePackets().HasTag(kStringSavedModelPath); - RET_CHECK(has_exactly_one_model) - << "Must have exactly one of saved model filepath in options or " - "input_side_packets STRING_MODEL_FILE_PATH"; - // Path of savedmodel. - if (cc->InputSidePackets().HasTag(kStringSavedModelPath)) { - cc->InputSidePackets().Tag(kStringSavedModelPath).Set(); - } - // A TensorFlow model loaded and ready for use along with tensor - cc->OutputSidePackets().Tag("SESSION").Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const auto& options = - cc->Options(); - std::string path = cc->InputSidePackets().HasTag(kStringSavedModelPath) - ? cc->InputSidePackets() - .Tag(kStringSavedModelPath) - .Get() - : options.saved_model_path(); - if (options.load_latest_model()) { - RET_CHECK_OK(GetLatestDirectory(&path)); - } - - // Set user specified tags properly. - // If no tags specified will use tensorflow::kSavedModelTagServe by default. - std::unordered_set tags_set; - for (const std::string& tag : options.saved_model_tag()) { - tags_set.insert(tag); - } - if (tags_set.empty()) { - tags_set.insert(tensorflow::kSavedModelTagServe); - } - - tensorflow::RunOptions run_options; - tensorflow::SessionOptions session_options; - session_options.config = options.session_config(); - auto saved_model = absl::make_unique(); - ::tensorflow::Status status = tensorflow::LoadSavedModel( - session_options, run_options, path, tags_set, saved_model.get()); - if (!status.ok()) { - return absl::Status(static_cast(status.code()), - status.ToString()); - } - - auto session = absl::make_unique(); - session->session = std::move(saved_model->session); - - RET_CHECK(!options.signature_name().empty()); - const auto& signature_def_map = saved_model->meta_graph_def.signature_def(); - const auto& signature_def = signature_def_map.at(options.signature_name()); - for (const auto& input_signature : signature_def.inputs()) { - session->tag_to_tensor_map[MaybeConvertSignatureToTag( - input_signature.first, options)] = input_signature.second.name(); - } - for (const auto& output_signature : signature_def.outputs()) { - session->tag_to_tensor_map[MaybeConvertSignatureToTag( - output_signature.first, options)] = output_signature.second.name(); - } - - cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release())); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(TensorFlowSessionFromSavedModelCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto deleted file mode 100644 index 927d3b51f..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "tensorflow/core/protobuf/config.proto"; - -message TensorFlowSessionFromSavedModelCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TensorFlowSessionFromSavedModelCalculatorOptions ext = 244429915; - } - // TODO: SessionBundles provided global step versioning of models - // that let you load the latest model. If there's a similar solution for - // SavedModels, include a flag to load the most recent model. - - // Path to a directory containing a trained TensorFlow model as prepared - // by SavedModel. - optional string saved_model_path = 1; - // The name of the generic signature to load into the mapping from tags to - // tensor names. - optional string signature_name = 2 [default = "serving_default"]; - // Whether to convert the signature keys to uppercase as well as switch /'s - // and -'s to _'s, which enables common signatures to be used as Tags. - optional bool convert_signature_to_tags = 3 [default = true]; - // If true, saved_model_path can have multiple exported models in - // subdirectories saved_model_path/%08d and the alphabetically last (i.e., - // latest checkpoint) model is loaded. Note that saved models are not exported - // in numbered directories by default. If you want to use this feature, you - // need to arrange your directories by global_step or some other order when - // you save your models. - optional bool load_latest_model = 4; - // [DEPRECATED] If true, this calculator will try to initialize local Tensor - // Processing Unit (TPU) hardware so that the Tensorflow session loaded from - // this saved model may benefit from TPU speedups. If you want to use this - // feature, you need to make sure that the calculator runs on a machine that - // has TPU hardware installed. The saved model should have correct device - // placements in the graph (have the ops already placed on TPU), typically if - // the saved model was exported through TPUEstimator then device placement is - // automatically taken care of. - optional bool use_tpu = 5 [deprecated = true]; - // User specified tags in a saved model. - // If no tag is specified, then use "serve" as the default. Note that in order - // to use TPU accelerator hardware, the tag "tpu" needs to be specified. - repeated string saved_model_tag = 6; - - // Tensorflow session config options. - optional tensorflow.ConfigProto session_config = 7; -} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc deleted file mode 100644 index 7016f14bb..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/tag_map_helper.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/device_attributes.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -std::string GetSavedModelDir() { - std::string out_path = - file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/", - "tensorflow_saved_model/00000000"); - return out_path; -} - -// Helper function that creates Tensor INT32 matrix with size 1x3. -tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) { - tf::Tensor tensor(tf::DT_INT32, - tf::TensorShape(std::vector({1, 3}))); - auto matrix = tensor.matrix(); - matrix(0, 0) = v1; - matrix(0, 1) = v2; - matrix(0, 2) = v3; - return tensor; -} - -class TensorFlowSessionFromSavedModelCalculatorTest : public ::testing::Test { - protected: - void SetUp() override { - extendable_options_.Clear(); - options_ = extendable_options_.MutableExtension( - TensorFlowSessionFromSavedModelCalculatorOptions::ext); - options_->set_saved_model_path(GetSavedModelDir()); - } - - CalculatorOptions extendable_options_; - TensorFlowSessionFromSavedModelCalculatorOptions* options_; -}; - -TEST_F(TensorFlowSessionFromSavedModelCalculatorTest, - CreatesPacketWithGraphAndBindings) { - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromSavedModelCalculator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { - $0 - } - })", - options_->DebugString())); - MP_ASSERT_OK(runner.Run()); - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); - - // Bindings are inserted. - EXPECT_EQ(session.tag_to_tensor_map.size(), 4); - - // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators. - EXPECT_FALSE(session.tag_to_tensor_map.find("A") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("B") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("EXPENSIVE") == - session.tag_to_tensor_map.end()); - // Sanity: find() actually returns a reference to end() if element not - // found. - EXPECT_TRUE(session.tag_to_tensor_map.find("Z") == - session.tag_to_tensor_map.end()); - - EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("EXPENSIVE"), "expensive:0"); -} - -TEST_F(TensorFlowSessionFromSavedModelCalculatorTest, - CreateSessionFromSidePacket) { - options_->clear_saved_model_path(); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromSavedModelCalculator" - input_side_packet: "STRING_SAVED_MODEL_PATH:saved_model_dir" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { - $0 - } - })", - options_->DebugString())); - runner.MutableSidePackets()->Tag("STRING_SAVED_MODEL_PATH") = - MakePacket(GetSavedModelDir()); - MP_ASSERT_OK(runner.Run()); - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); -} - -// Integration test. Verifies that TensorFlowInferenceCalculator correctly -// consumes the Packet emitted by this factory. -TEST_F(TensorFlowSessionFromSavedModelCalculatorTest, - ProducesPacketUsableByTensorFlowInferenceCalculator) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - node { - calculator: "TensorFlowInferenceCalculator" - input_side_packet: "SESSION:tf_model" - input_stream: "A:a_tensor" - output_stream: "MULTIPLIED:multiplied_tensor" - options { - [mediapipe.TensorFlowInferenceCalculatorOptions.ext] { - batch_size: 5 - add_batch_dim_to_tensors: false - } - } - } - node { - calculator: "TensorFlowSessionFromSavedModelCalculator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { - $0 - } - } - } - input_stream: "a_tensor" - )", - options_->DebugString())); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("multiplied_tensor"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "a_tensor", - Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("a_tensor")); - - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - // input tensor gets multiplied by [[3, 2, 1]]. Expected output: - tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10); - EXPECT_EQ(expected_multiplication.DebugString(), - packet.Get().DebugString()); - - ASSERT_FALSE(poller.Next(&packet)); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TensorFlowSessionFromSavedModelCalculatorTest, - GetsBundleGivenParentDirectory) { - options_->set_saved_model_path( - std::string(file::SplitPath(GetSavedModelDir()).first)); - options_->set_load_latest_model(true); - - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromSavedModelCalculator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { - $0 - } - })", - options_->DebugString())); - MP_ASSERT_OK(runner.Run()); - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); -} - -TEST_F(TensorFlowSessionFromSavedModelCalculatorTest, - ConfiguresSessionGivenConfig) { - options_->set_saved_model_path( - std::string(file::SplitPath(GetSavedModelDir()).first)); - options_->set_load_latest_model(true); - options_->mutable_session_config()->mutable_device_count()->insert( - {"CPU", 10}); - CalculatorRunner runner(absl::Substitute(R"( - calculator: "TensorFlowSessionFromSavedModelCalculator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelCalculatorOptions.ext]: { - $0 - } - })", - options_->DebugString())); - MP_ASSERT_OK(runner.Run()); - const TensorFlowSession& session = - runner.OutputSidePackets().Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); - std::vector devices; - ASSERT_EQ(session.session->ListDevices(&devices), tensorflow::Status::OK()); - EXPECT_THAT(devices.size(), 10); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc deleted file mode 100644 index 6489b0267..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#if !defined(__ANDROID__) -#include "mediapipe/framework/port/file_helpers.h" -#endif -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/packet_generator.h" -#include "mediapipe/framework/packet_type.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/status_util.h" -#include "tensorflow/cc/saved_model/constants.h" -#include "tensorflow/cc/saved_model/loader.h" -#include "tensorflow/cc/saved_model/tag_constants.h" - -namespace mediapipe { - -namespace { -static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH"; - -// Given the path to a directory containing multiple tensorflow saved models -// in subdirectories, replaces path with the alphabetically last subdirectory. -absl::Status GetLatestDirectory(std::string* path) { -#if defined(__ANDROID__) - return absl::UnimplementedError( - "GetLatestDirectory is not implemented on Android"); -#else - std::vector saved_models; - RET_CHECK_OK(file::MatchInTopSubdirectories( - *path, tensorflow::kSavedModelFilenamePb, &saved_models)); - RET_CHECK_GT(saved_models.size(), 0) - << "No exported bundles found in " << path; - ::std::sort(saved_models.begin(), saved_models.end()); - *path = std::string(file::Dirname(saved_models.back())); - return absl::OkStatus(); -#endif -} - -// If options.convert_signature_to_tags() is set, will convert letters to -// uppercase and replace /'s and -'s with _'s. This enables the standard -// SavedModel classification, regression, and prediction signatures to be used -// as uppercase INPUTS and OUTPUTS tags for streams and supports other common -// patterns. -const std::string MaybeConvertSignatureToTag( - const std::string& name, - const TensorFlowSessionFromSavedModelGeneratorOptions& options) { - if (options.convert_signature_to_tags()) { - std::string output; - output.resize(name.length()); - std::transform(name.begin(), name.end(), output.begin(), - [](unsigned char c) { return std::toupper(c); }); - output = absl::StrReplaceAll(output, {{"/", "_"}}); - output = absl::StrReplaceAll(output, {{"-", "_"}}); - return output; - } else { - return name; - } -} - -} // namespace - -// TensorFlowSessionFromSavedModelGenerator is a MediaPipe packet generator -// that loads a trained TensorFlow model exported via SavedModel's exporter and -// returns a Packet containing a unique_ptr to a mediapipe::TensorFlowSession, -// which in turn contains a TensorFlow Session ready for execution and a map -// between tags and tensor names. -class TensorFlowSessionFromSavedModelGenerator : public PacketGenerator { - public: - static absl::Status FillExpectations( - const PacketGeneratorOptions& extendable_options, - PacketTypeSet* input_side_packets, PacketTypeSet* output_side_packets) { - const TensorFlowSessionFromSavedModelGeneratorOptions& options = - extendable_options.GetExtension( - TensorFlowSessionFromSavedModelGeneratorOptions::ext); - const bool has_exactly_one_model = - options.saved_model_path().empty() == - input_side_packets->HasTag(kStringSavedModelPath); - RET_CHECK(has_exactly_one_model) - << "Must have exactly one of saved model filepath in options or " - "input_side_packets STRING_MODEL_FILE_PATH"; - // Path of savedmodel. - if (input_side_packets->HasTag(kStringSavedModelPath)) { - input_side_packets->Tag(kStringSavedModelPath).Set(); - } - // A TensorFlow model loaded and ready for use along with tensor - output_side_packets->Tag("SESSION").Set(); - return absl::OkStatus(); - } - - static absl::Status Generate(const PacketGeneratorOptions& extendable_options, - const PacketSet& input_side_packets, - PacketSet* output_side_packets) { - const TensorFlowSessionFromSavedModelGeneratorOptions& options = - extendable_options.GetExtension( - TensorFlowSessionFromSavedModelGeneratorOptions::ext); - std::string path = - input_side_packets.HasTag(kStringSavedModelPath) - ? input_side_packets.Tag(kStringSavedModelPath).Get() - : options.saved_model_path(); - if (options.load_latest_model()) { - RET_CHECK_OK(GetLatestDirectory(&path)); - } - - // Set user specified tags properly. - // If no tags specified will use tensorflow::kSavedModelTagServe by default. - std::unordered_set tags_set; - for (const std::string& tag : options.saved_model_tag()) { - tags_set.insert(tag); - } - if (tags_set.empty()) { - tags_set.insert(tensorflow::kSavedModelTagServe); - } - - tensorflow::RunOptions run_options; - tensorflow::SessionOptions session_options; - session_options.config = options.session_config(); - auto saved_model = absl::make_unique(); - ::tensorflow::Status status = tensorflow::LoadSavedModel( - session_options, run_options, path, tags_set, saved_model.get()); - if (!status.ok()) { - return absl::Status(static_cast(status.code()), - status.ToString()); - } - auto session = absl::make_unique(); - session->session = std::move(saved_model->session); - - RET_CHECK(!options.signature_name().empty()); - const auto& signature_def_map = saved_model->meta_graph_def.signature_def(); - const auto& signature_def = signature_def_map.at(options.signature_name()); - for (const auto& input_signature : signature_def.inputs()) { - session->tag_to_tensor_map[MaybeConvertSignatureToTag( - input_signature.first, options)] = input_signature.second.name(); - } - for (const auto& output_signature : signature_def.outputs()) { - session->tag_to_tensor_map[MaybeConvertSignatureToTag( - output_signature.first, options)] = output_signature.second.name(); - } - - output_side_packets->Tag("SESSION") = Adopt(session.release()); - return absl::OkStatus(); - } -}; -REGISTER_PACKET_GENERATOR(TensorFlowSessionFromSavedModelGenerator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto deleted file mode 100644 index d24a1cd73..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/packet_generator.proto"; -import "tensorflow/core/protobuf/config.proto"; - -message TensorFlowSessionFromSavedModelGeneratorOptions { - extend mediapipe.PacketGeneratorOptions { - optional TensorFlowSessionFromSavedModelGeneratorOptions ext = 151486368; - } - // TODO: SessionBundles provided global step versioning of models - // that let you load the latest model. If there's a similar solution for - // SavedModels, include a flag to load the most recent model. - - // Path to a directory containing a trained TensorFlow model as prepared - // by SavedModel. - optional string saved_model_path = 1; - // The name of the generic signature to load into the mapping from tags to - // tensor names. - optional string signature_name = 2 [default = "serving_default"]; - // Whether to convert the signature keys to uppercase as well as switch /'s - // and -'s to _'s, which enables common signatures to be used as Tags. - optional bool convert_signature_to_tags = 3 [default = true]; - // If true, saved_model_path can have multiple exported models in - // subdirectories saved_model_path/%08d and the alphabetically last (i.e., - // latest checkpoint) model is loaded. Note that saved models are not exported - // in numbered directories by default. If you want to use this feature, you - // need to arrange your directories by global_step or some other order when - // you save your models. - optional bool load_latest_model = 4; - // [DEPRECATED] If true, this calculator will try to initialize local Tensor - // Processing Unit (TPU) hardware so that the Tensorflow session loaded from - // this saved model may benefit from TPU speedups. If you want to use this - // feature, you need to make sure that the calculator runs on a machine that - // has TPU hardware installed. The saved model should have correct device - // placements in the graph (have the ops already placed on TPU), typically if - // the saved model was exported through TPUEstimator then device placement is - // automatically taken care of. - optional bool use_tpu = 5 [deprecated = true]; - // User specified tags in a saved model. - // If no tag is specified, then use "serve" as the default. Note that in order - // to use TPU accelerator hardware, the tag "tpu" needs to be specified. - repeated string saved_model_tag = 6; - - // Tensorflow session config options. - optional tensorflow.ConfigProto session_config = 9; -} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc deleted file mode 100644 index aca506f0b..000000000 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session.h" -#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/packet_generator.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/tag_map_helper.h" -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/core/framework/device_attributes.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -std::string GetSavedModelDir() { - std::string out_path = - file::JoinPath("./", "mediapipe/calculators/tensorflow/testdata/", - "tensorflow_saved_model/00000000"); - return out_path; -} - -// Helper function that creates Tensor INT32 matrix with size 1x3. -tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) { - tf::Tensor tensor(tf::DT_INT32, - tf::TensorShape(std::vector({1, 3}))); - auto matrix = tensor.matrix(); - matrix(0, 0) = v1; - matrix(0, 1) = v2; - matrix(0, 2) = v3; - return tensor; -} - -class TensorFlowSessionFromSavedModelGeneratorTest : public ::testing::Test { - protected: - void SetUp() override { - extendable_options_.Clear(); - generator_options_ = extendable_options_.MutableExtension( - TensorFlowSessionFromSavedModelGeneratorOptions::ext); - generator_options_->set_saved_model_path(GetSavedModelDir()); - } - - PacketGeneratorOptions extendable_options_; - TensorFlowSessionFromSavedModelGeneratorOptions* generator_options_; -}; - -TEST_F(TensorFlowSessionFromSavedModelGeneratorTest, - CreatesPacketWithGraphAndBindings) { - PacketSet input_side_packets(tool::CreateTagMap({}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromSavedModelGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - const TensorFlowSession& session = - output_side_packets.Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); - - // Bindings are inserted. - EXPECT_EQ(session.tag_to_tensor_map.size(), 4); - - // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators. - EXPECT_FALSE(session.tag_to_tensor_map.find("A") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("B") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") == - session.tag_to_tensor_map.end()); - EXPECT_FALSE(session.tag_to_tensor_map.find("EXPENSIVE") == - session.tag_to_tensor_map.end()); - // Sanity: find() actually returns a reference to end() if element not - // found. - EXPECT_TRUE(session.tag_to_tensor_map.find("Z") == - session.tag_to_tensor_map.end()); - - EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0"); - EXPECT_EQ(session.tag_to_tensor_map.at("EXPENSIVE"), "expensive:0"); -} - -TEST_F(TensorFlowSessionFromSavedModelGeneratorTest, - CreateSessionFromSidePacket) { - generator_options_->clear_saved_model_path(); - PacketSet input_side_packets( - tool::CreateTagMap({"STRING_SAVED_MODEL_PATH:saved_model_dir"}).value()); - input_side_packets.Tag("STRING_SAVED_MODEL_PATH") = - Adopt(new std::string(GetSavedModelDir())); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromSavedModelGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - const TensorFlowSession& session = - output_side_packets.Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); -} - -// Integration test. Verifies that TensorFlowInferenceCalculator correctly -// consumes the Packet emitted by this factory. -TEST_F(TensorFlowSessionFromSavedModelGeneratorTest, - ProducesPacketUsableByTensorFlowInferenceCalculator) { - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - node { - calculator: "TensorFlowInferenceCalculator" - input_side_packet: "SESSION:tf_model" - input_stream: "A:a_tensor" - output_stream: "MULTIPLIED:multiplied_tensor" - options { - [mediapipe.TensorFlowInferenceCalculatorOptions.ext] { - batch_size: 5 - add_batch_dim_to_tensors: false - } - } - } - - packet_generator { - packet_generator: "TensorFlowSessionFromSavedModelGenerator" - output_side_packet: "SESSION:tf_model" - options { - [mediapipe.TensorFlowSessionFromSavedModelGeneratorOptions.ext]: { - $0 - } - } - } - input_stream: "a_tensor" - )", - generator_options_->DebugString())); - - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("multiplied_tensor"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "a_tensor", - Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("a_tensor")); - - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - // input tensor gets multiplied by [[3, 2, 1]]. Expected output: - tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10); - EXPECT_EQ(expected_multiplication.DebugString(), - packet.Get().DebugString()); - - ASSERT_FALSE(poller.Next(&packet)); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TensorFlowSessionFromSavedModelGeneratorTest, - GetsBundleGivenParentDirectory) { - generator_options_->set_saved_model_path( - std::string(file::SplitPath(GetSavedModelDir()).first)); - generator_options_->set_load_latest_model(true); - - PacketSet input_side_packets(tool::CreateTagMap({}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromSavedModelGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - const TensorFlowSession& session = - output_side_packets.Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); -} - -TEST_F(TensorFlowSessionFromSavedModelGeneratorTest, - ConfiguresSessionGivenConfig) { - generator_options_->set_saved_model_path( - std::string(file::SplitPath(GetSavedModelDir()).first)); - generator_options_->set_load_latest_model(true); - generator_options_->mutable_session_config()->mutable_device_count()->insert( - {"CPU", 10}); - - PacketSet input_side_packets(tool::CreateTagMap({}).value()); - PacketSet output_side_packets( - tool::CreateTagMap({"SESSION:session"}).value()); - absl::Status run_status = tool::RunGenerateAndValidateTypes( - "TensorFlowSessionFromSavedModelGenerator", extendable_options_, - input_side_packets, &output_side_packets); - MP_EXPECT_OK(run_status) << run_status.message(); - const TensorFlowSession& session = - output_side_packets.Tag("SESSION").Get(); - // Session must be set. - ASSERT_NE(session.session, nullptr); - std::vector devices; - ASSERT_EQ(session.session->ListDevices(&devices), tensorflow::Status::OK()); - EXPECT_THAT(devices.size(), 10); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/checkpoint b/mediapipe/calculators/tensorflow/testdata/bundle/00000000/checkpoint deleted file mode 100755 index 055afb948..000000000 --- a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/checkpoint +++ /dev/null @@ -1,2 +0,0 @@ -model_checkpoint_path: "export-?????-of-00001" -all_model_checkpoint_paths: "export-?????-of-00001" diff --git a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export-00000-of-00001 b/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export-00000-of-00001 deleted file mode 100755 index 82e808c48..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export-00000-of-00001 and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export.meta b/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export.meta deleted file mode 100755 index 1a5b318d4..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/bundle/00000000/export.meta and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/frozen_graph_def.pb b/mediapipe/calculators/tensorflow/testdata/frozen_graph_def.pb deleted file mode 100755 index 966790d52..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/frozen_graph_def.pb and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/model.chkpt-0 b/mediapipe/calculators/tensorflow/testdata/model.chkpt-0 deleted file mode 100755 index 82e808c48..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/model.chkpt-0 and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/model.chkpt-0.meta b/mediapipe/calculators/tensorflow/testdata/model.chkpt-0.meta deleted file mode 100755 index c21a47f8e..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/model.chkpt-0.meta and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/saved_model.pb b/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/saved_model.pb deleted file mode 100755 index b66128177..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/saved_model.pb and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.data-00000-of-00001 b/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.data-00000-of-00001 deleted file mode 100755 index b3be9e288..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.index b/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.index deleted file mode 100755 index a2e8d586f..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/tensorflow_saved_model/00000000/variables/variables.index and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/testdata/tf_graph_def.pb b/mediapipe/calculators/tensorflow/testdata/tf_graph_def.pb deleted file mode 100755 index bb046494a..000000000 Binary files a/mediapipe/calculators/tensorflow/testdata/tf_graph_def.pb and /dev/null differ diff --git a/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc b/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc deleted file mode 100644 index 28271f3a7..000000000 --- a/mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/io/record_reader.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/file_system.h" - -namespace mediapipe { - -const char kTFRecordPath[] = "TFRECORD_PATH"; -const char kRecordIndex[] = "RECORD_INDEX"; -const char kExampleTag[] = "EXAMPLE"; -const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; - -// Reads a tensorflow example/sequence example from a tfrecord file. -// If the "RECORD_INDEX" input side packet is provided, the calculator is going -// to fetch the example/sequence example of the tfrecord file at the target -// record index. Otherwise, the reader always reads the first example/sequence -// example of the tfrecord file. -// -// Example config: -// node { -// calculator: "TFRecordReaderCalculator" -// input_side_packet: "TFRECORD_PATH:tfrecord_path" -// input_side_packet: "RECORD_INDEX:record_index" -// output_side_packet: "SEQUENCE_EXAMPLE:sequence_example" -// } -class TFRecordReaderCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; - -absl::Status TFRecordReaderCalculator::GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag(kTFRecordPath).Set(); - if (cc->InputSidePackets().HasTag(kRecordIndex)) { - cc->InputSidePackets().Tag(kRecordIndex).Set(); - } - - RET_CHECK(cc->OutputSidePackets().HasTag(kExampleTag) || - cc->OutputSidePackets().HasTag(kSequenceExampleTag)) - << "TFRecordReaderCalculator must output either Tensorflow example or " - "sequence example."; - if (cc->OutputSidePackets().HasTag(kExampleTag)) { - cc->OutputSidePackets().Tag(kExampleTag).Set(); - } else { - cc->OutputSidePackets() - .Tag(kSequenceExampleTag) - .Set(); - } - return absl::OkStatus(); -} - -absl::Status TFRecordReaderCalculator::Open(CalculatorContext* cc) { - std::unique_ptr file; - auto tf_status = tensorflow::Env::Default()->NewRandomAccessFile( - cc->InputSidePackets().Tag(kTFRecordPath).Get(), &file); - RET_CHECK(tf_status.ok()) - << "Failed to open tfrecord file: " << tf_status.ToString(); - tensorflow::io::RecordReader reader(file.get(), - tensorflow::io::RecordReaderOptions()); - tensorflow::uint64 offset = 0; - tensorflow::tstring example_str; - const int target_idx = - cc->InputSidePackets().HasTag(kRecordIndex) - ? cc->InputSidePackets().Tag(kRecordIndex).Get() - : 0; - int current_idx = 0; - while (current_idx <= target_idx) { - tf_status = reader.ReadRecord(&offset, &example_str); - RET_CHECK(tf_status.ok()) - << "Failed to read tfrecord: " << tf_status.ToString(); - if (current_idx == target_idx) { - if (cc->OutputSidePackets().HasTag(kExampleTag)) { - tensorflow::Example tf_example; - tf_example.ParseFromArray(example_str.data(), example_str.size()); - cc->OutputSidePackets() - .Tag(kExampleTag) - .Set(MakePacket(std::move(tf_example))); - } else { - tensorflow::SequenceExample tf_sequence_example; - tf_sequence_example.ParseFromString(example_str); - cc->OutputSidePackets() - .Tag(kSequenceExampleTag) - .Set(MakePacket( - std::move(tf_sequence_example))); - } - } - ++current_idx; - } - - return absl::OkStatus(); -} - -absl::Status TFRecordReaderCalculator::Process(CalculatorContext* cc) { - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(TFRecordReaderCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc deleted file mode 100644 index c4f7d40a9..000000000 --- a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc +++ /dev/null @@ -1,489 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/flat_hash_map.h" -#include "absl/strings/match.h" -#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/audio_decoder.pb.h" -#include "mediapipe/util/sequence/media_sequence.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" - -namespace mediapipe { - -// Streams: -const char kBBoxTag[] = "BBOX"; -const char kImageTag[] = "IMAGE"; -const char kKeypointsTag[] = "KEYPOINTS"; -const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_"; -const char kForwardFlowImageTag[] = "FORWARD_FLOW_ENCODED"; - -// Side Packets: -const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; -const char kDatasetRootDirTag[] = "DATASET_ROOT"; -const char kDataPath[] = "DATA_PATH"; -const char kPacketResamplerOptions[] = "RESAMPLER_OPTIONS"; -const char kImagesFrameRateTag[] = "IMAGE_FRAME_RATE"; -const char kAudioDecoderOptions[] = "AUDIO_DECODER_OPTIONS"; - -namespace tf = ::tensorflow; -namespace mpms = mediapipe::mediasequence; - -// Source calculator to unpack side_packets and streams from tf.SequenceExamples -// -// Often, only side_packets or streams need to be output, but both can be output -// if needed. A tf.SequenceExample always needs to be supplied as an -// input_side_packet. The SequenceExample must be in the format described in -// media_sequence.h. This documentation will first describe the side_packets -// the calculator can output, and then describe the streams. -// -// Side_packets are commonly used to specify which clip to extract data from. -// Seeking into a video does not necessarily provide consistent timestamps when -// resampling to a known rate. To enable consistent timestamps, we unpack the -// metadata into options for the MediaDecoderCalculator and the -// PacketResamplerCalculator. To ensure consistent timestamps, the MediaDecoder -// needs to seek to slightly before the clip starts, so it sees at least one -// packet before the first packet we want to keep. The PacketResamplerCalculator -// then trims down the timestamps. Furthermore, we should always specify that we -// want timestamps from a base timestamp of 0, so we have the same resampled -// frames after a seek that we would have from the start of a video. In summary, -// when decoding image frames, output both the DECODER_OPTIONS and -// RESAMPLER_OPTIONS. In the base_media_decoder_options, specify which streams -// you want. In the base_packet_resampler_options, specify the frame_rate you -// want and base_timestamp = 0. In the options for this calculator, specify -// padding extra_padding_from_media_decoder such that at least one frame arrives -// before the first frame the PacketResamplerCalculator should output. -// -// Optional output_side_packets include (referenced by tag): -// DATA_PATH: The data_path context feature joined onto the -// options.dataset_root_directory or input_side_packet of DATASET_ROOT. -// RESAMPLER_OPTIONS: CalculatorOptions to pass to the -// PacketResamplerCalculator. The most accurate procedure for sampling a -// range of frames is to request a padded time range from the -// MediaDecoderCalculator and then trim it down to the proper time range with -// the PacketResamplerCalculator. -// IMAGES_FRAME_RATE: The frame rate of the images in the original video as a -// double. -// -// Example config: -// node { -// calculator: "UnpackMediaSequenceCalculator" -// input_side_packet: "SEQUENCE_EXAMPLE:example_input_side_packet" -// input_side_packet: "DATASET_ROOT:path_to_dataset_root_directory" -// output_side_packet: "DATA_PATH:full_path_to_data_element" -// output_side_packet: "RESAMPLER_OPTIONS:packet_resampler_options" -// options { -// [mediapipe.UnpackMediaSequenceCalculatorOptions.ext]: { -// base_packet_resampler_options { -// frame_rate: 1.0 # PARAM_FRAME_RATE -// base_timestamp: 0 -// } -// } -// } -// } -// -// The calculator also takes a tf.SequenceExample as a side input and outputs -// the data in streams from the SequenceExample at the proper timestamps. The -// SequenceExample must conform to the description in media_sequence.h. -// Timestamps in the SequenceExample must be in sequential order. -// -// The following output stream tags are supported: -// IMAGE: encoded images as strings. (IMAGE_${NAME} is supported.) -// FORWARD_FLOW_ENCODED: encoded FORWARD_FLOW prefix images as strings. -// FLOAT_FEATURE_${NAME}: the feature named ${NAME} as vector. -// BBOX: bounding boxes as vectors. (BBOX_${NAME} is supported.) -// -// Example config: -// node { -// calculator: "UnpackMediaSequenceCalculator" -// input_side_packet: "SEQUENCE_EXAMPLE:example_input_side_packet" -// output_stream: "IMAGE:frames" -// output_stream: "FLOAT_FEATURE_FDENSE:fdense_vf" -// output_stream: "BBOX:faces" -// } -class UnpackMediaSequenceCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - RET_CHECK(cc->InputSidePackets().HasTag(kSequenceExampleTag)); - cc->InputSidePackets().Tag(kSequenceExampleTag).Set(); - // Optional side inputs. - if (cc->InputSidePackets().HasTag(kDatasetRootDirTag)) { - cc->InputSidePackets().Tag(kDatasetRootDirTag).Set(); - } - if (cc->OutputSidePackets().HasTag(kDataPath)) { - cc->OutputSidePackets().Tag(kDataPath).Set(); - } - if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions)) { - cc->OutputSidePackets() - .Tag(kAudioDecoderOptions) - .Set(); - } - if (cc->OutputSidePackets().HasTag(kImagesFrameRateTag)) { - cc->OutputSidePackets().Tag(kImagesFrameRateTag).Set(); - } - if (cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) { - cc->OutputSidePackets() - .Tag(kPacketResamplerOptions) - .Set(); - } - if ((options.has_padding_before_label() || - options.has_padding_after_label()) && - !(cc->OutputSidePackets().HasTag(kAudioDecoderOptions) || - cc->OutputSidePackets().HasTag(kPacketResamplerOptions))) { - return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "If specifying padding, must output " << kPacketResamplerOptions - << "or" << kAudioDecoderOptions; - } - - if (cc->Outputs().HasTag(kForwardFlowImageTag)) { - cc->Outputs().Tag(kForwardFlowImageTag).Set(); - } - for (const auto& tag : cc->Outputs().GetTags()) { - if (absl::StartsWith(tag, kImageTag)) { - std::string key = ""; - if (tag != kImageTag) { - int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kImageTag)_?" - } - } - cc->Outputs().Tag(tag).Set(); - } - if (absl::StartsWith(tag, kBBoxTag)) { - std::string key = ""; - if (tag != kBBoxTag) { - int tag_length = sizeof(kBBoxTag) / sizeof(*kBBoxTag) - 1; - if (tag[tag_length] == '_') { - key = tag.substr(tag_length + 1); - } else { - continue; // Skip keys that don't match "(kBBoxTag)_?" - } - } - cc->Outputs().Tag(tag).Set>(); - } - if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) { - cc->Outputs().Tag(tag).Set>(); - } - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - // Copy the packet to copy the otherwise inaccessible shared ptr. - example_packet_holder_ = cc->InputSidePackets().Tag(kSequenceExampleTag); - sequence_ = &example_packet_holder_.Get(); - - // Collect the timestamps for all streams keyed by the timestamp feature's - // key. While creating this data structure we also identify the last - // timestamp and the associated feature. This information is used in process - // to output batches of packets in order. - timestamps_.clear(); - int64 last_timestamp_seen = Timestamp::PreStream().Value(); - first_timestamp_seen_ = Timestamp::OneOverPostStream().Value(); - for (const auto& map_kv : sequence_->feature_lists().feature_list()) { - if (absl::StrContains(map_kv.first, "/timestamp")) { - LOG(INFO) << "Found feature timestamps: " << map_kv.first - << " with size: " << map_kv.second.feature_size(); - int64 recent_timestamp = Timestamp::PreStream().Value(); - for (int i = 0; i < map_kv.second.feature_size(); ++i) { - int64 next_timestamp = - mpms::GetInt64sAt(*sequence_, map_kv.first, i).Get(0); - RET_CHECK_GT(next_timestamp, recent_timestamp) - << "Timestamps must be sequential. If you're seeing this message " - << "you may have added images to the same SequenceExample twice. " - << "Key: " << map_kv.first; - timestamps_[map_kv.first].push_back(next_timestamp); - recent_timestamp = next_timestamp; - if (recent_timestamp < first_timestamp_seen_) { - first_timestamp_seen_ = recent_timestamp; - } - } - if (recent_timestamp > last_timestamp_seen && - recent_timestamp < Timestamp::PostStream().Value()) { - last_timestamp_key_ = map_kv.first; - last_timestamp_seen = recent_timestamp; - } - } - } - if (!timestamps_.empty()) { - for (const auto& kv : timestamps_) { - if (!kv.second.empty() && - kv.second[0] < Timestamp::PostStream().Value()) { - // These checks only make sense if any values are not PostStream, but - // only need to be made once. - RET_CHECK(!last_timestamp_key_.empty()) - << "Something went wrong because the timestamp key is unset. " - << "Example: " << sequence_->DebugString(); - RET_CHECK_GT(last_timestamp_seen, Timestamp::PreStream().Value()) - << "Something went wrong because the last timestamp is unset. " - << "Example: " << sequence_->DebugString(); - RET_CHECK_LT(first_timestamp_seen_, - Timestamp::OneOverPostStream().Value()) - << "Something went wrong because the first timestamp is unset. " - << "Example: " << sequence_->DebugString(); - break; - } - } - } - current_timestamp_index_ = 0; - process_poststream_ = false; - - // Determine the data path and output it. - const auto& options = cc->Options(); - const auto& sequence = cc->InputSidePackets() - .Tag(kSequenceExampleTag) - .Get(); - if (cc->OutputSidePackets().HasTag(kDataPath)) { - std::string root_directory = ""; - if (cc->InputSidePackets().HasTag(kDatasetRootDirTag)) { - root_directory = - cc->InputSidePackets().Tag(kDatasetRootDirTag).Get(); - } else if (options.has_dataset_root_directory()) { - root_directory = options.dataset_root_directory(); - } - - std::string data_path = mpms::GetClipDataPath(sequence); - if (!root_directory.empty()) { - if (root_directory[root_directory.size() - 1] == '/') { - data_path = root_directory + data_path; - } else { - data_path = root_directory + "/" + data_path; - } - } - cc->OutputSidePackets().Tag(kDataPath).Set( - MakePacket(data_path)); - } - - // Set the start and end of the clip in the appropriate options protos. - double start_time = 0; - double end_time = 0; - if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions) || - cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) { - if (mpms::HasClipStartTimestamp(sequence)) { - start_time = - Timestamp(mpms::GetClipStartTimestamp(sequence)).Seconds() - - options.padding_before_label(); - } - if (mpms::HasClipEndTimestamp(sequence)) { - end_time = Timestamp(mpms::GetClipEndTimestamp(sequence)).Seconds() + - options.padding_after_label(); - } - } - if (cc->OutputSidePackets().HasTag(kAudioDecoderOptions)) { - auto audio_decoder_options = absl::make_unique( - options.base_audio_decoder_options()); - if (mpms::HasClipStartTimestamp(sequence)) { - if (options.force_decoding_from_start_of_media()) { - audio_decoder_options->set_start_time(0); - } else { - audio_decoder_options->set_start_time( - start_time - options.extra_padding_from_media_decoder()); - } - } - if (mpms::HasClipEndTimestamp(sequence)) { - audio_decoder_options->set_end_time( - end_time + options.extra_padding_from_media_decoder()); - } - LOG(INFO) << "Created AudioDecoderOptions:\n" - << audio_decoder_options->DebugString(); - cc->OutputSidePackets() - .Tag(kAudioDecoderOptions) - .Set(Adopt(audio_decoder_options.release())); - } - if (cc->OutputSidePackets().HasTag(kPacketResamplerOptions)) { - auto resampler_options = absl::make_unique(); - *(resampler_options->MutableExtension( - PacketResamplerCalculatorOptions::ext)) = - options.base_packet_resampler_options(); - if (mpms::HasClipStartTimestamp(sequence)) { - resampler_options - ->MutableExtension(PacketResamplerCalculatorOptions::ext) - ->set_start_time(Timestamp::FromSeconds(start_time).Value()); - } - if (mpms::HasClipEndTimestamp(sequence)) { - resampler_options - ->MutableExtension(PacketResamplerCalculatorOptions::ext) - ->set_end_time(Timestamp::FromSeconds(end_time).Value()); - } - - LOG(INFO) << "Created PacketResamplerOptions:\n" - << resampler_options->DebugString(); - cc->OutputSidePackets() - .Tag(kPacketResamplerOptions) - .Set(Adopt(resampler_options.release())); - } - - // Output the remaining side outputs. - if (cc->OutputSidePackets().HasTag(kImagesFrameRateTag)) { - cc->OutputSidePackets() - .Tag(kImagesFrameRateTag) - .Set(MakePacket(mpms::GetImageFrameRate(sequence))); - } - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (timestamps_.empty()) { - // This occurs when we only have metadata to unpack. - LOG(INFO) << "only unpacking metadata because there are no timestamps."; - return tool::StatusStop(); - } - // In Process(), we loop through timestamps on a reference stream and emit - // all packets on all streams that have a timestamp between the current - // reference timestep and the previous reference timestep. This ensures that - // we emit all timestamps in order, but also only emit a limited number in - // any particular call to Process(). At the every end, we output the - // poststream packets. If we only have poststream packets, - // last_timestamp_key_ will be empty. - int64 start_timestamp = 0; - int64 end_timestamp = 0; - if (last_timestamp_key_.empty() || process_poststream_) { - process_poststream_ = true; - start_timestamp = Timestamp::PostStream().Value(); - end_timestamp = Timestamp::OneOverPostStream().Value(); - } else { - start_timestamp = - timestamps_[last_timestamp_key_][current_timestamp_index_]; - if (current_timestamp_index_ == 0) { - start_timestamp = first_timestamp_seen_; - } - - end_timestamp = start_timestamp + 1; // Base case at end of sequence. - if (current_timestamp_index_ < - timestamps_[last_timestamp_key_].size() - 1) { - end_timestamp = - timestamps_[last_timestamp_key_][current_timestamp_index_ + 1]; - } - } - - for (const auto& map_kv : timestamps_) { - for (int i = 0; i < map_kv.second.size(); ++i) { - if (map_kv.second[i] >= start_timestamp && - map_kv.second[i] < end_timestamp) { - const Timestamp current_timestamp = - map_kv.second[i] == Timestamp::PostStream().Value() - ? Timestamp::PostStream() - : Timestamp(map_kv.second[i]); - - if (absl::StrContains(map_kv.first, mpms::GetImageTimestampKey())) { - std::vector pieces = absl::StrSplit(map_kv.first, '/'); - std::string feature_key = ""; - std::string possible_tag = kImageTag; - if (pieces[0] != "image") { - feature_key = pieces[0]; - possible_tag = absl::StrCat(kImageTag, "_", feature_key); - } - if (cc->Outputs().HasTag(possible_tag)) { - cc->Outputs() - .Tag(possible_tag) - .Add(new std::string( - mpms::GetImageEncodedAt(feature_key, *sequence_, i)), - current_timestamp); - } - } - - if (cc->Outputs().HasTag(kForwardFlowImageTag) && - map_kv.first == mpms::GetForwardFlowTimestampKey()) { - cc->Outputs() - .Tag(kForwardFlowImageTag) - .Add(new std::string( - mpms::GetForwardFlowEncodedAt(*sequence_, i)), - current_timestamp); - } - if (absl::StrContains(map_kv.first, mpms::GetBBoxTimestampKey())) { - std::vector pieces = absl::StrSplit(map_kv.first, '/'); - std::string feature_key = ""; - std::string possible_tag = kBBoxTag; - if (pieces[0] != "region") { - feature_key = pieces[0]; - possible_tag = absl::StrCat(kBBoxTag, "_", feature_key); - } - if (cc->Outputs().HasTag(possible_tag)) { - const auto& bboxes = mpms::GetBBoxAt(feature_key, *sequence_, i); - cc->Outputs() - .Tag(possible_tag) - .Add(new std::vector(bboxes.begin(), bboxes.end()), - current_timestamp); - } - } - - if (absl::StrContains(map_kv.first, "feature")) { - std::vector pieces = absl::StrSplit(map_kv.first, '/'); - RET_CHECK_GT(pieces.size(), 1) - << "Failed to parse the feature substring before / from key " - << map_kv.first; - std::string feature_key = pieces[0]; - std::string possible_tag = kFloatFeaturePrefixTag + feature_key; - if (cc->Outputs().HasTag(possible_tag)) { - const auto& float_list = - mpms::GetFeatureFloatsAt(feature_key, *sequence_, i); - cc->Outputs() - .Tag(possible_tag) - .Add(new std::vector(float_list.begin(), - float_list.end()), - current_timestamp); - } - } - } - } - } - - ++current_timestamp_index_; - if (current_timestamp_index_ < timestamps_[last_timestamp_key_].size()) { - return absl::OkStatus(); - } else { - if (process_poststream_) { - // Once we've processed the PostStream timestamp we can stop. - return tool::StatusStop(); - } else { - // Otherwise, we still need to do one more pass to process it. - process_poststream_ = true; - return absl::OkStatus(); - } - } - } - - // Hold a copy of the packet to prevent the shared_ptr from dying and then - // access the SequenceExample with a handy pointer. - const tf::SequenceExample* sequence_; - Packet example_packet_holder_; - - // Store a map from the keys for each stream to the timestamps for each - // key. This allows us to identify which packets to output for each stream - // for timestamps within a given time window. - std::map> timestamps_; - // Store the stream with the latest timestamp in the SequenceExample. - std::string last_timestamp_key_; - // Store the index of the current timestamp. Will be less than - // timestamps_[last_timestamp_key_].size(). - int current_timestamp_index_; - // Store the very first timestamp, so we output everything on the first frame. - int64 first_timestamp_seen_; - // List of keypoint names. - std::vector keypoint_names_; - // Default keypoint location when missing. - float default_keypoint_location_; - bool process_poststream_; -}; -REGISTER_CALCULATOR(UnpackMediaSequenceCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto deleted file mode 100644 index 51cc870c7..000000000 --- a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.proto +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/calculators/core/packet_resampler_calculator.proto"; -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/audio_decoder.proto"; - -message UnpackMediaSequenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional UnpackMediaSequenceCalculatorOptions ext = 244411537; - } - - // Path to the root directory of the data set that SequenceExample directory - // paths are relative from. If present, the input_side_packet overrides this - // value. - optional string dataset_root_directory = 1; - - // Time in seconds to pad before (or after) timestamps in context's - // clip/timestamp/start and clip/timestamp/end. These settings modify the - // clip's time range in the base_media_decoder_options. - optional float padding_before_label = 3; - optional float padding_after_label = 4; - - // Time in seconds to apply as additional padding to the media decoder, but - // not to the packet resampler. - optional float extra_padding_from_media_decoder = 5 [default = 0.0]; - - // Stores the packet resampler settings for the graph. The most accurate - // procedure for sampling a range of frames is to request a padded time range - // from the MediaDecoderCalculator and then trim it down to the proper time - // range with the PacketResamplerCalculator. - optional PacketResamplerCalculatorOptions base_packet_resampler_options = 6; - - // Decode media from time zero. This setting overrides other padding - // parameters for the MediaDecoderCalculator. End time parameters are still - // respected. - optional bool force_decoding_from_start_of_media = 7; - - // Stores the audio decoder settings for the graph. (e.g. which audio - // stream to pull from the video.) The sequence's metadata overrides - // the clip start and end times and outputs these for the - // AudioDecoderCalculator to consume. - optional AudioDecoderOptions base_audio_decoder_options = 9; -} diff --git a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc deleted file mode 100644 index e8e40bad3..000000000 --- a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc +++ /dev/null @@ -1,637 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/memory/memory.h" -#include "absl/strings/numbers.h" -#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" -#include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/rectangle.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/audio_decoder.pb.h" -#include "mediapipe/util/sequence/media_sequence.h" -#include "tensorflow/core/example/example.pb.h" - -namespace mediapipe { -namespace { - -namespace tf = ::tensorflow; -namespace mpms = mediapipe::mediasequence; - -class UnpackMediaSequenceCalculatorTest : public ::testing::Test { - protected: - void SetUpCalculator(const std::vector& output_streams, - const std::vector& output_side_packets, - const std::vector& input_side_packets = {}, - const CalculatorOptions* options = nullptr) { - CalculatorGraphConfig::Node config; - config.set_calculator("UnpackMediaSequenceCalculator"); - config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence"); - for (const std::string& stream : output_streams) { - config.add_output_stream(stream); - } - for (const std::string& side_packet : output_side_packets) { - config.add_output_side_packet(side_packet); - } - for (const std::string& side_packet : input_side_packets) { - config.add_input_side_packet(side_packet); - } - if (options != nullptr) { - *config.mutable_options() = *options; - } - LOG(INFO) << config.DebugString(); - runner_ = absl::make_unique(config); - } - - void SetUp() override { - sequence_ = absl::make_unique(); - mpms::SetClipMediaId(video_id_, sequence_.get()); - mpms::SetClipDataPath(data_path_, sequence_.get()); - mpms::SetClipStartTimestamp(start_time_, sequence_.get()); - mpms::SetClipEndTimestamp(end_time_, sequence_.get()); - mpms::SetClipEncodedMediaBytes(encoded_video_data_, sequence_.get()); - mpms::SetClipEncodedMediaStartTimestamp(encoded_video_start_timestamp_, - sequence_.get()); - mpms::SetImageFrameRate(image_frame_rate_, sequence_.get()); - } - - std::unique_ptr sequence_; - std::unique_ptr runner_; - const std::string video_id_ = "test_video_id"; - const std::string data_path_ = "test_directory"; - const int64 start_time_ = 3000000; - const int64 end_time_ = 5000000; - const std::string encoded_video_data_ = "encoded_video_data"; - const int64 encoded_video_start_timestamp_ = 1000000; - const double image_frame_rate_ = 1.0; -}; - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneImage) { - SetUpCalculator({"IMAGE:images"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - - int num_images = 1; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(i, input_sequence.get()); - mpms::AddImageEncoded(test_image_string, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("IMAGE").packets; - ASSERT_EQ(num_images, output_packets.size()); - - for (int i = 0; i < num_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoImages) { - SetUpCalculator({"IMAGE:images"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(i, input_sequence.get()); - mpms::AddImageEncoded(test_image_string, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("IMAGE").packets; - ASSERT_EQ(num_images, output_packets.size()); - - for (int i = 0; i < num_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPrefixedImages) { - std::string prefix = "PREFIX"; - SetUpCalculator({"IMAGE_PREFIX:images"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(prefix, i, input_sequence.get()); - mpms::AddImageEncoded(prefix, test_image_string, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("IMAGE_PREFIX").packets; - ASSERT_EQ(num_images, output_packets.size()); - - for (int i = 0; i < num_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksOneForwardFlowImage) { - SetUpCalculator({"FORWARD_FLOW_ENCODED:flow_images"}, {}); - auto input_sequence = absl::make_unique(); - const std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - const std::string test_image_string = "test_image_string"; - const int num_forward_flow_images = 1; - for (int i = 0; i < num_forward_flow_images; ++i) { - mpms::AddForwardFlowTimestamp(i, input_sequence.get()); - mpms::AddForwardFlowEncoded(test_image_string, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets; - ASSERT_EQ(num_forward_flow_images, output_packets.size()); - - for (int i = 0; i < num_forward_flow_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - ASSERT_EQ(output_packets[i].Timestamp().Value(), static_cast(i)); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoForwardFlowImages) { - SetUpCalculator({"FORWARD_FLOW_ENCODED:flow_images"}, {}); - auto input_sequence = absl::make_unique(); - const std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - const std::string test_image_strings[2] = {"test_image_string0", - "test_image_string1"}; - const int num_forward_flow_images = 2; - for (int i = 0; i < num_forward_flow_images; ++i) { - mpms::AddForwardFlowTimestamp(i, input_sequence.get()); - mpms::AddForwardFlowEncoded(test_image_strings[i], input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("FORWARD_FLOW_ENCODED").packets; - ASSERT_EQ(num_forward_flow_images, output_packets.size()); - - for (int i = 0; i < num_forward_flow_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_strings[i]); - ASSERT_EQ(output_packets[i].Timestamp().Value(), static_cast(i)); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksBBoxes) { - SetUpCalculator({"BBOX:test", "FLOAT_FEATURE_OTHER:other"}, {}); - auto input_sequence = absl::make_unique(); - - std::vector> bboxes = { - {Location::CreateRelativeBBoxLocation(0.1, 0.2, 0.7, 0.7), - Location::CreateRelativeBBoxLocation(0.3, 0.4, 0.2, 0.1)}, - {Location::CreateRelativeBBoxLocation(0.2, 0.3, 0.4, 0.5)}}; - - for (int i = 0; i < bboxes.size(); ++i) { - mpms::AddBBox(bboxes[i], input_sequence.get()); - mpms::AddBBoxTimestamp(i, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("BBOX").packets; - ASSERT_EQ(bboxes.size(), output_packets.size()); - - for (int i = 0; i < bboxes.size(); ++i) { - const auto& output_vector = - output_packets[i].Get>(); - for (int j = 0; j < bboxes[i].size(); ++j) { - ASSERT_EQ(output_vector[j].GetRelativeBBox(), - bboxes[i][j].GetRelativeBBox()); - } - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPrefixedBBoxes) { - std::string prefix = "PREFIX"; - SetUpCalculator({"BBOX_PREFIX:test", "FLOAT_FEATURE_OTHER:other"}, {}); - auto input_sequence = absl::make_unique(); - - std::vector> bboxes = { - {Location::CreateRelativeBBoxLocation(0.1, 0.2, 0.7, 0.7), - Location::CreateRelativeBBoxLocation(0.3, 0.4, 0.2, 0.1)}, - {Location::CreateRelativeBBoxLocation(0.2, 0.3, 0.4, 0.5)}}; - - for (int i = 0; i < bboxes.size(); ++i) { - mpms::AddBBox(prefix, bboxes[i], input_sequence.get()); - mpms::AddBBoxTimestamp(prefix, i, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("BBOX_PREFIX").packets; - ASSERT_EQ(bboxes.size(), output_packets.size()); - - for (int i = 0; i < bboxes.size(); ++i) { - const auto& output_vector = - output_packets[i].Get>(); - for (int j = 0; j < bboxes[i].size(); ++j) { - ASSERT_EQ(output_vector[j].GetRelativeBBox(), - bboxes[i][j].GetRelativeBBox()); - } - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoFloatLists) { - SetUpCalculator({"FLOAT_FEATURE_TEST:test", "FLOAT_FEATURE_OTHER:other"}, {}); - auto input_sequence = absl::make_unique(); - - int num_float_lists = 2; - for (int i = 0; i < num_float_lists; ++i) { - std::vector data(2, 2 << i); - mpms::AddFeatureFloats("TEST", data, input_sequence.get()); - mpms::AddFeatureFloats("OTHER", data, input_sequence.get()); - mpms::AddFeatureTimestamp("TEST", i, input_sequence.get()); - mpms::AddFeatureTimestamp("OTHER", i, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("FLOAT_FEATURE_TEST").packets; - ASSERT_EQ(num_float_lists, output_packets.size()); - - for (int i = 0; i < num_float_lists; ++i) { - const auto& output_vector = output_packets[i].Get>(); - ASSERT_THAT(output_vector, - ::testing::ElementsAreArray(std::vector(2, 2 << i))); - } - - const std::vector& output_packets_other = - runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets; - ASSERT_EQ(num_float_lists, output_packets_other.size()); - - for (int i = 0; i < num_float_lists; ++i) { - const auto& output_vector = - output_packets_other[i].Get>(); - ASSERT_THAT(output_vector, - ::testing::ElementsAreArray(std::vector(2, 2 << i))); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksNonOverlappingTimestamps) { - SetUpCalculator({"IMAGE:images", "FLOAT_FEATURE_OTHER:other"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - int num_images = 2; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(i, input_sequence.get()); - mpms::AddImageEncoded(test_image_string, input_sequence.get()); - } - int num_float_lists = 2; - for (int i = 0; i < num_float_lists; ++i) { - std::vector data(2, 2 << i); - mpms::AddFeatureFloats("OTHER", data, input_sequence.get()); - mpms::AddFeatureTimestamp("OTHER", i + 5, input_sequence.get()); - } - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("IMAGE").packets; - ASSERT_EQ(num_images, output_packets.size()); - - for (int i = 0; i < num_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - } - - const std::vector& output_packets_other = - runner_->Outputs().Tag("FLOAT_FEATURE_OTHER").packets; - ASSERT_EQ(num_float_lists, output_packets_other.size()); - - for (int i = 0; i < num_float_lists; ++i) { - const auto& output_vector = - output_packets_other[i].Get>(); - ASSERT_THAT(output_vector, - ::testing::ElementsAreArray(std::vector(2, 2 << i))); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksTwoPostStreamFloatLists) { - SetUpCalculator( - {"FLOAT_FEATURE_FDENSE_AVG:avg", "FLOAT_FEATURE_FDENSE_MAX:max"}, {}); - auto input_sequence = absl::make_unique(); - mpms::AddFeatureFloats("FDENSE_AVG", {1.0f, 2.0f}, input_sequence.get()); - mpms::AddFeatureTimestamp("FDENSE_AVG", Timestamp::PostStream().Value(), - input_sequence.get()); - - mpms::AddFeatureFloats("FDENSE_MAX", {3.0f, 4.0f}, input_sequence.get()); - mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(), - input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - MP_ASSERT_OK(runner_->Run()); - - const std::vector& fdense_avg_packets = - runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_AVG").packets; - ASSERT_EQ(fdense_avg_packets.size(), 1); - const auto& fdense_avg_vector = - fdense_avg_packets[0].Get>(); - ASSERT_THAT(fdense_avg_vector, ::testing::ElementsAreArray({1.0f, 2.0f})); - ASSERT_THAT(fdense_avg_packets[0].Timestamp(), - ::testing::Eq(Timestamp::PostStream())); - - const std::vector& fdense_max_packets = - runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets; - ASSERT_EQ(fdense_max_packets.size(), 1); - const auto& fdense_max_vector = - fdense_max_packets[0].Get>(); - ASSERT_THAT(fdense_max_vector, ::testing::ElementsAreArray({3.0f, 4.0f})); - ASSERT_THAT(fdense_max_packets[0].Timestamp(), - ::testing::Eq(Timestamp::PostStream())); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksImageWithPostStreamFloatList) { - SetUpCalculator({"IMAGE:images"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - - int num_images = 1; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(i, input_sequence.get()); - mpms::AddImageEncoded(test_image_string, input_sequence.get()); - } - - mpms::AddFeatureFloats("FDENSE_MAX", {3.0f, 4.0f}, input_sequence.get()); - mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(), - input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& output_packets = - runner_->Outputs().Tag("IMAGE").packets; - ASSERT_EQ(num_images, output_packets.size()); - - for (int i = 0; i < num_images; ++i) { - const std::string& output_image = output_packets[i].Get(); - ASSERT_EQ(output_image, test_image_string); - } -} - -TEST_F(UnpackMediaSequenceCalculatorTest, UnpacksPostStreamFloatListWithImage) { - SetUpCalculator({"FLOAT_FEATURE_FDENSE_MAX:max"}, {}); - auto input_sequence = absl::make_unique(); - std::string test_video_id = "test_video_id"; - mpms::SetClipMediaId(test_video_id, input_sequence.get()); - - std::string test_image_string = "test_image_string"; - - int num_images = 1; - for (int i = 0; i < num_images; ++i) { - mpms::AddImageTimestamp(i, input_sequence.get()); - mpms::AddImageEncoded(test_image_string, input_sequence.get()); - } - - mpms::AddFeatureFloats("FDENSE_MAX", {3.0f, 4.0f}, input_sequence.get()); - mpms::AddFeatureTimestamp("FDENSE_MAX", Timestamp::PostStream().Value(), - input_sequence.get()); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(input_sequence.release()); - - MP_ASSERT_OK(runner_->Run()); - - const std::vector& fdense_max_packets = - runner_->Outputs().Tag("FLOAT_FEATURE_FDENSE_MAX").packets; - ASSERT_EQ(fdense_max_packets.size(), 1); - const auto& fdense_max_vector = - fdense_max_packets[0].Get>(); - ASSERT_THAT(fdense_max_vector, ::testing::ElementsAreArray({3.0f, 4.0f})); - ASSERT_THAT(fdense_max_packets[0].Timestamp(), - ::testing::Eq(Timestamp::PostStream())); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromPacket) { - SetUpCalculator({}, {"DATA_PATH:data_path"}, {"DATASET_ROOT:root"}); - - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - - std::string root = "test_root"; - runner_->MutableSidePackets()->Tag("DATASET_ROOT") = PointToForeign(&root); - MP_ASSERT_OK(runner_->Run()); - - MP_ASSERT_OK(runner_->OutputSidePackets() - .Tag("DATA_PATH") - .ValidateAsType()); - ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get(), - root + "/" + data_path_); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromOptions) { - CalculatorOptions options; - std::string root = "test_root"; - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_dataset_root_directory(root); - SetUpCalculator({}, {"DATA_PATH:data_path"}, {}, &options); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - - MP_ASSERT_OK(runner_->Run()); - - MP_ASSERT_OK(runner_->OutputSidePackets() - .Tag("DATA_PATH") - .ValidateAsType()); - ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get(), - root + "/" + data_path_); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetDatasetFromExample) { - SetUpCalculator({}, {"DATA_PATH:data_path"}); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - MP_ASSERT_OK(runner_->Run()); - - MP_ASSERT_OK(runner_->OutputSidePackets() - .Tag("DATA_PATH") - .ValidateAsType()); - ASSERT_EQ(runner_->OutputSidePackets().Tag("DATA_PATH").Get(), - data_path_); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptions) { - CalculatorOptions options; - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_before_label(1); - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_after_label(2); - SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {}, - &options); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - MP_ASSERT_OK(runner_->Run()); - - MP_EXPECT_OK(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .ValidateAsType()); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .Get() - .start_time(), - 2.0, 1e-5); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .Get() - .end_time(), - 7.0, 1e-5); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetAudioDecoderOptionsOverride) { - CalculatorOptions options; - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_before_label(1); - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_after_label(2); - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_force_decoding_from_start_of_media(true); - SetUpCalculator({}, {"AUDIO_DECODER_OPTIONS:audio_decoder_options"}, {}, - &options); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - MP_ASSERT_OK(runner_->Run()); - - MP_EXPECT_OK(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .ValidateAsType()); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .Get() - .start_time(), - 0.0, 1e-5); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("AUDIO_DECODER_OPTIONS") - .Get() - .end_time(), - 7.0, 1e-5); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetPacketResamplingOptions) { - // TODO: Suport proto3 proto.Any in CalculatorOptions. - // TODO: Avoid proto2 extensions in "RESAMPLER_OPTIONS". - CalculatorOptions options; - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_before_label(1); - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->set_padding_after_label(2); - options.MutableExtension(UnpackMediaSequenceCalculatorOptions::ext) - ->mutable_base_packet_resampler_options() - ->set_frame_rate(1.0); - SetUpCalculator({}, {"RESAMPLER_OPTIONS:resampler_options"}, {}, &options); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - MP_ASSERT_OK(runner_->Run()); - - MP_EXPECT_OK(runner_->OutputSidePackets() - .Tag("RESAMPLER_OPTIONS") - .ValidateAsType()); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("RESAMPLER_OPTIONS") - .Get() - .GetExtension(PacketResamplerCalculatorOptions::ext) - .start_time(), - 2000000, 1); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("RESAMPLER_OPTIONS") - .Get() - .GetExtension(PacketResamplerCalculatorOptions::ext) - .end_time(), - 7000000, 1); - EXPECT_NEAR(runner_->OutputSidePackets() - .Tag("RESAMPLER_OPTIONS") - .Get() - .GetExtension(PacketResamplerCalculatorOptions::ext) - .frame_rate(), - 1.0, 1e-5); -} - -TEST_F(UnpackMediaSequenceCalculatorTest, GetFrameRateFromExample) { - SetUpCalculator({}, {"IMAGE_FRAME_RATE:frame_rate"}); - runner_->MutableSidePackets()->Tag("SEQUENCE_EXAMPLE") = - Adopt(sequence_.release()); - MP_ASSERT_OK(runner_->Run()); - MP_EXPECT_OK(runner_->OutputSidePackets() - .Tag("IMAGE_FRAME_RATE") - .ValidateAsType()); - EXPECT_EQ(runner_->OutputSidePackets().Tag("IMAGE_FRAME_RATE").Get(), - image_frame_rate_); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc b/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc deleted file mode 100644 index efb3037f8..000000000 --- a/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "tensorflow/core/example/example.pb.h" -#include "tensorflow/core/example/feature.pb.h" - -namespace mediapipe { -namespace { - -const char kId[] = "id"; -const char kRgb[] = "rgb"; -const char kAudio[] = "audio"; -const char kDesiredSegmentSize[] = "DESIRED_SEGMENT_SIZE"; -const char kYt8mId[] = "YT8M_ID"; -const char kYt8mSequenceExample[] = "YT8M_SEQUENCE_EXAMPLE"; -const char kQuantizedRgbFeature[] = "QUANTIZED_RGB_FEATURE"; -const char kQuantizedAudioFeature[] = "QUANTIZED_AUDIO_FEATURE"; -const char kSegmentSize[] = "SEGMENT_SIZE"; -const char kLappedTensorBufferCalculatorOptions[] = - "LAPPED_TENSOR_BUFFER_CALCULATOR_OPTIONS"; - -std::string GetQuantizedFeature( - const tensorflow::SequenceExample& sequence_example, const std::string& key, - int index) { - const auto& bytes_list = sequence_example.feature_lists() - .feature_list() - .at(key) - .feature() - .Get(index) - .bytes_list() - .value(); - CHECK_EQ(1, bytes_list.size()); - return bytes_list.Get(0); -} -} // namespace - -// Unpacks YT8M Sequence Example. Note that the audio feature and rgb feature -// output are quantized. DequantizeByteArrayCalculator can do the dequantization -// for you. -// -// Example config: -// node { -// calculator: "UnpackYt8mSequenceExampleCalculator" -// input_side_packet: "YT8M_SEQUENCE_EXAMPLE:yt8m_sequence_example" -// output_stream: "QUANTIZED_RGB_FEATURE:quantized_rgb_feature" -// output_stream: "QUANTIZED_AUDIO_FEATURE:quantized_audio_feature" -// } -class UnpackYt8mSequenceExampleCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets() - .Tag(kYt8mSequenceExample) - .Set(); - if (cc->InputSidePackets().HasTag(kDesiredSegmentSize)) { - cc->InputSidePackets().Tag(kDesiredSegmentSize).Set(); - } - cc->Outputs().Tag(kQuantizedRgbFeature).Set(); - cc->Outputs().Tag(kQuantizedAudioFeature).Set(); - if (cc->OutputSidePackets().HasTag(kYt8mId)) { - cc->OutputSidePackets().Tag(kYt8mId).Set(); - } - if (cc->OutputSidePackets().HasTag(kLappedTensorBufferCalculatorOptions)) { - cc->OutputSidePackets() - .Tag(kLappedTensorBufferCalculatorOptions) - .Set<::mediapipe::LappedTensorBufferCalculatorOptions>(); - } - if (cc->OutputSidePackets().HasTag(kSegmentSize)) { - cc->OutputSidePackets().Tag(kSegmentSize).Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const tensorflow::SequenceExample& sequence_example = - cc->InputSidePackets() - .Tag(kYt8mSequenceExample) - .Get(); - const std::string& yt8m_id = - sequence_example.context().feature().at(kId).bytes_list().value().Get( - 0); - if (cc->OutputSidePackets().HasTag(kYt8mId)) { - cc->OutputSidePackets().Tag(kYt8mId).Set( - MakePacket(yt8m_id)); - } - - int rgb_feature_list_length = - sequence_example.feature_lists().feature_list().at(kRgb).feature_size(); - int audio_feature_list_length = sequence_example.feature_lists() - .feature_list() - .at(kAudio) - .feature_size(); - - if (rgb_feature_list_length != audio_feature_list_length) { - return absl::FailedPreconditionError(absl::StrCat( - "Data corruption: the length of audio features and rgb features are " - "not equal. Please check the sequence example that contains yt8m " - "id: ", - yt8m_id)); - } - feature_list_length_ = rgb_feature_list_length; - if (cc->OutputSidePackets().HasTag(kLappedTensorBufferCalculatorOptions) || - cc->OutputSidePackets().HasTag(kSegmentSize)) { - // If the desired segment size is specified, take the min of the length of - // the feature list and the desired size to be the output segment size. - int segment_size = feature_list_length_; - if (cc->InputSidePackets().HasTag(kDesiredSegmentSize)) { - int desired_segment_size = - cc->InputSidePackets().Tag(kDesiredSegmentSize).Get(); - RET_CHECK(desired_segment_size > 0) - << "The desired segment size must be greater than zero."; - segment_size = std::min( - feature_list_length_, - cc->InputSidePackets().Tag(kDesiredSegmentSize).Get()); - } - if (cc->OutputSidePackets().HasTag( - kLappedTensorBufferCalculatorOptions)) { - auto lapped_tensor_buffer_calculator_options = absl::make_unique< - ::mediapipe::LappedTensorBufferCalculatorOptions>(); - lapped_tensor_buffer_calculator_options->set_add_batch_dim_to_tensors( - true); - lapped_tensor_buffer_calculator_options->set_buffer_size(segment_size); - lapped_tensor_buffer_calculator_options->set_overlap(segment_size - 1); - lapped_tensor_buffer_calculator_options->set_timestamp_offset( - segment_size - 1); - cc->OutputSidePackets() - .Tag(kLappedTensorBufferCalculatorOptions) - .Set(Adopt(lapped_tensor_buffer_calculator_options.release())); - } - if (cc->OutputSidePackets().HasTag(kSegmentSize)) { - cc->OutputSidePackets() - .Tag(kSegmentSize) - .Set(MakePacket(segment_size)); - } - } - LOG(INFO) << "Reading the sequence example that contains yt8m id: " - << yt8m_id << ". Feature list length: " << feature_list_length_; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (current_index_ >= feature_list_length_) { - return mediapipe::tool::StatusStop(); - } - const tensorflow::SequenceExample& sequence_example = - cc->InputSidePackets() - .Tag(kYt8mSequenceExample) - .Get(); - - // Uses microsecond as the unit of time. In the YT8M dataset, each feature - // represents a second. - const Timestamp timestamp = Timestamp(current_index_ * 1000000); - cc->Outputs() - .Tag(kQuantizedRgbFeature) - .AddPacket( - MakePacket( - GetQuantizedFeature(sequence_example, kRgb, current_index_)) - .At(timestamp)); - cc->Outputs() - .Tag(kQuantizedAudioFeature) - .AddPacket( - MakePacket( - GetQuantizedFeature(sequence_example, kAudio, current_index_)) - .At(timestamp)); - ++current_index_; - return absl::OkStatus(); - } - - private: - int current_index_ = 0; - int feature_list_length_ = 0; -}; - -REGISTER_CALCULATOR(UnpackYt8mSequenceExampleCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc deleted file mode 100644 index 96208b3e5..000000000 --- a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Converts vector (or vector>) to 1D (or 2D) tf::Tensor. - -#include "mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace { -auto& INPUT_1D = VectorFloatToTensorCalculatorOptions::INPUT_1D; -auto& INPUT_2D = VectorFloatToTensorCalculatorOptions::INPUT_2D; -} // namespace - -namespace tf = ::tensorflow; - -// The calculator expects one input (a packet containing a vector or -// vector>) and generates one output (a packet containing a -// tf::Tensor containing the same data). The output tensor will be either -// 1D or 2D with dimensions corresponding to the input vector float. -// It will hold DT_FLOAT values. -// -// Example config: -// node { -// calculator: "VectorFloatToTensorCalculator" -// input_stream: "vector_float_features" -// output_stream: "tensor_features" -// } -class VectorFloatToTensorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - VectorFloatToTensorCalculatorOptions options_; -}; -REGISTER_CALCULATOR(VectorFloatToTensorCalculator); - -absl::Status VectorFloatToTensorCalculator::GetContract( - CalculatorContract* cc) { - const auto& options = cc->Options(); - // Start with only one input packet. - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - if (options.input_size() == INPUT_2D) { - cc->Inputs().Index(0).Set>>( - /* "Input vector>." */); - } else if (options.input_size() == INPUT_1D) { - cc->Inputs().Index(0).Set>( - // Output vector. - ); - } else { - LOG(FATAL) << "input size not supported"; - } - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Index(0).Set( - // Output stream with data as tf::Tensor and the same TimeSeriesHeader. - ); - return absl::OkStatus(); -} - -absl::Status VectorFloatToTensorCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - cc->SetOffset(0); - return absl::OkStatus(); -} - -absl::Status VectorFloatToTensorCalculator::Process(CalculatorContext* cc) { - tf::TensorShape tensor_shape; - if (options_.input_size() == INPUT_2D) { - const std::vector>& input = - cc->Inputs().Index(0).Value().Get>>(); - - const int32 rows = input.size(); - RET_CHECK_GE(rows, 1); - const int32 cols = input[0].size(); - RET_CHECK_GE(cols, 1); - for (int i = 1; i < rows; ++i) { - RET_CHECK_EQ(input[i].size(), cols); - } - if (options_.transpose()) { - tensor_shape = tf::TensorShape({cols, rows}); - } else { - tensor_shape = tf::TensorShape({rows, cols}); - } - auto output = ::absl::make_unique(tf::DT_FLOAT, tensor_shape); - for (int r = 0; r < rows; ++r) { - for (int c = 0; c < cols; ++c) { - if (options_.transpose()) { - output->tensor()(c, r) = input[r][c]; - } else { - output->tensor()(r, c) = input[r][c]; - } - } - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else if (options_.input_size() == INPUT_1D) { - const std::vector& input = - cc->Inputs().Index(0).Value().Get>(); - RET_CHECK_GE(input.size(), 1); - const int32 length = input.size(); - tensor_shape = tf::TensorShape({length}); - auto output = ::absl::make_unique(tf::DT_FLOAT, tensor_shape); - for (int i = 0; i < length; ++i) { - output->tensor()(i) = input.at(i); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else { - LOG(FATAL) << "input size not supported"; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.proto b/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.proto deleted file mode 100644 index 01be3b72c..000000000 --- a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message VectorFloatToTensorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional VectorFloatToTensorCalculatorOptions ext = 136399889; - } - enum InputSize { - UNKNOWN = 0; - INPUT_1D = 1; - INPUT_2D = 2; - } - - // If input_size is INPUT_2D, unpack a vector> to a - // 2d tensor (matrix). If INPUT_1D, - // convert a vector into a 1d tensor (vector). - optional InputSize input_size = 1 [default = INPUT_1D]; - - // If true, the output tensor is transposed. - // Otherwise, the output tensor is not transposed. - // It will be ignored if tensor_is_2d is INPUT_1D. - optional bool transpose = 2 [default = false]; -} diff --git a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_test.cc b/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_test.cc deleted file mode 100644 index aadce3615..000000000 --- a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_test.cc +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class VectorToTensorFloatCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner( - const VectorFloatToTensorCalculatorOptions::InputSize input_size, - const bool transpose) { - CalculatorGraphConfig::Node config; - config.set_calculator("VectorFloatToTensorCalculator"); - config.add_input_stream("input_float"); - config.add_output_stream("output_tensor"); - auto options = config.mutable_options()->MutableExtension( - VectorFloatToTensorCalculatorOptions::ext); - options->set_input_size(input_size); - options->set_transpose(transpose); - runner_ = ::absl::make_unique(config); - } - - void TestConvertFromVectoVectorFloat(const bool transpose) { - SetUpRunner(VectorFloatToTensorCalculatorOptions::INPUT_2D, transpose); - auto input = ::absl::make_unique>>( - 2, std::vector(2)); - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - // 2^i can be represented exactly in floating point numbers - // if 'i' is small. - input->at(i).at(j) = static_cast(1 << (i * 2 + j)); - } - } - - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(2, output_tensor.dims()); - EXPECT_EQ(tf::DT_FLOAT, output_tensor.dtype()); - const auto matrix = output_tensor.matrix(); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - if (!transpose) { - EXPECT_EQ(1 << (i * 2 + j), matrix(i, j)); - } else { - EXPECT_EQ(1 << (j * 2 + i), matrix(i, j)); - } - } - } - } - - std::unique_ptr runner_; -}; - -TEST_F(VectorToTensorFloatCalculatorTest, ConvertsFromVectorFloat) { - SetUpRunner(VectorFloatToTensorCalculatorOptions::INPUT_1D, false); - auto input = ::absl::make_unique>(5); - for (int i = 0; i < 5; ++i) { - // 2^i can be represented exactly in floating point numbers if 'i' is small. - input->at(i) = static_cast(1 << i); - } - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_FLOAT, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(1 << i, vec(i)); - } -} - -TEST_F(VectorToTensorFloatCalculatorTest, ConvertsFromVectorVectorFloat) { - for (bool transpose : {false, true}) { - TestConvertFromVectoVectorFloat(transpose); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc deleted file mode 100644 index f5bf7661e..000000000 --- a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Converts a single int or vector or vector> to 1D (or 2D) -// tf::Tensor. - -#include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -const char kVectorInt[] = "VECTOR_INT"; -const char kSingleInt[] = "SINGLE_INT"; -const char kTensorOut[] = "TENSOR_OUT"; - -namespace { -auto& INPUT_1D = VectorIntToTensorCalculatorOptions::INPUT_1D; -auto& INPUT_2D = VectorIntToTensorCalculatorOptions::INPUT_2D; -} // namespace - -namespace tf = ::tensorflow; - -template -void AssignMatrixValue(int r, int c, int value, tf::Tensor* output_tensor) { - output_tensor->tensor()(r, c) = value; -} - -// The calculator expects one input (a packet containing a single int or -// vector or vector>) and generates one output (a packet -// containing a tf::Tensor containing the same data). The output tensor will be -// either 1D or 2D with dimensions corresponding to the input vector int. It -// will hold DT_INT32 or DT_UINT8 or DT_INT64 values. -// -// Example config: -// node { -// calculator: "VectorIntToTensorCalculator" -// input_stream: "SINGLE_INT:segment_size_int_stream" -// output_stream: "TENSOR_OUT:segment_size_tensor" -// } -// -// or -// -// node { -// calculator: "VectorIntToTensorCalculator" -// input_stream: "VECTOR_INT:vector_int_features" -// output_stream: "TENSOR_OUT:tensor_features" -// } -class VectorIntToTensorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - VectorIntToTensorCalculatorOptions options_; -}; -REGISTER_CALCULATOR(VectorIntToTensorCalculator); - -absl::Status VectorIntToTensorCalculator::GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - // Start with only one input packet. - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - if (options.input_size() == INPUT_2D) { - cc->Inputs().Tag(kVectorInt).Set>>(); - } else if (options.input_size() == INPUT_1D) { - if (cc->Inputs().HasTag(kSingleInt)) { - cc->Inputs().Tag(kSingleInt).Set(); - } else { - cc->Inputs().Tag(kVectorInt).Set>(); - } - } else { - LOG(FATAL) << "input size not supported"; - } - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Tag(kTensorOut).Set(); - return absl::OkStatus(); -} - -absl::Status VectorIntToTensorCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - RET_CHECK(options_.tensor_data_type() == tf::DT_UINT8 || - options_.tensor_data_type() == tf::DT_INT32 || - options_.tensor_data_type() == tf::DT_INT64) - << "Output tensor data type is not supported."; - return absl::OkStatus(); -} - -absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { - tf::TensorShape tensor_shape; - if (options_.input_size() == INPUT_2D) { - const std::vector>& input = - cc->Inputs() - .Tag(kVectorInt) - .Value() - .Get>>(); - - const int32 rows = input.size(); - CHECK_GE(rows, 1); - const int32 cols = input[0].size(); - CHECK_GE(cols, 1); - for (int i = 1; i < rows; ++i) { - CHECK_EQ(input[i].size(), cols); - } - if (options_.transpose()) { - tensor_shape = tf::TensorShape({cols, rows}); - } else { - tensor_shape = tf::TensorShape({rows, cols}); - } - auto output = ::absl::make_unique(options_.tensor_data_type(), - tensor_shape); - if (options_.transpose()) { - for (int r = 0; r < rows; ++r) { - for (int c = 0; c < cols; ++c) { - switch (options_.tensor_data_type()) { - case tf::DT_INT64: - AssignMatrixValue(c, r, input[r][c], output.get()); - break; - case tf::DT_UINT8: - AssignMatrixValue(c, r, input[r][c], output.get()); - break; - case tf::DT_INT32: - AssignMatrixValue(c, r, input[r][c], output.get()); - break; - default: - LOG(FATAL) << "tensor data type is not supported."; - } - } - } - } else { - for (int r = 0; r < rows; ++r) { - for (int c = 0; c < cols; ++c) { - switch (options_.tensor_data_type()) { - case tf::DT_INT64: - AssignMatrixValue(r, c, input[r][c], output.get()); - break; - case tf::DT_UINT8: - AssignMatrixValue(r, c, input[r][c], output.get()); - break; - case tf::DT_INT32: - AssignMatrixValue(r, c, input[r][c], output.get()); - break; - default: - LOG(FATAL) << "tensor data type is not supported."; - } - } - } - } - cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp()); - } else if (options_.input_size() == INPUT_1D) { - std::vector input; - if (cc->Inputs().HasTag(kSingleInt)) { - input.push_back(cc->Inputs().Tag(kSingleInt).Get()); - } else { - input = cc->Inputs().Tag(kVectorInt).Value().Get>(); - } - CHECK_GE(input.size(), 1); - const int32 length = input.size(); - tensor_shape = tf::TensorShape({length}); - auto output = ::absl::make_unique(options_.tensor_data_type(), - tensor_shape); - for (int i = 0; i < length; ++i) { - switch (options_.tensor_data_type()) { - case tf::DT_INT64: - output->tensor()(i) = input.at(i); - break; - case tf::DT_UINT8: - output->tensor()(i) = input.at(i); - break; - case tf::DT_INT32: - output->tensor()(i) = input.at(i); - break; - default: - LOG(FATAL) << "tensor data type is not supported."; - } - } - cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp()); - } else { - LOG(FATAL) << "input size not supported"; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto b/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto deleted file mode 100644 index 65554bb14..000000000 --- a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.proto +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "tensorflow/core/framework/types.proto"; - -message VectorIntToTensorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional VectorIntToTensorCalculatorOptions ext = 275364184; - } - enum InputSize { - UNKNOWN = 0; - INPUT_1D = 1; - INPUT_2D = 2; - } - - // If input_size is INPUT_2D, unpack a vector> to a - // 2d tensor (matrix). If INPUT_1D, convert a single int or vector - // into a 1d tensor (vector). - optional InputSize input_size = 1 [default = INPUT_1D]; - - // If true, the output tensor is transposed. - // Otherwise, the output tensor is not transposed. - // It will be ignored if tensor_is_2d is INPUT_1D. - optional bool transpose = 2 [default = false]; - - optional tensorflow.DataType tensor_data_type = 3 [default = DT_INT32]; -} diff --git a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc b/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc deleted file mode 100644 index 369c09660..000000000 --- a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_test.cc +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class VectorIntToTensorCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner( - const VectorIntToTensorCalculatorOptions::InputSize input_size, - const tensorflow::DataType tensor_data_type, const bool transpose, - const bool single_value) { - CalculatorGraphConfig::Node config; - config.set_calculator("VectorIntToTensorCalculator"); - if (single_value) { - config.add_input_stream("SINGLE_INT:input_int"); - } else { - config.add_input_stream("VECTOR_INT:input_int"); - } - config.add_output_stream("TENSOR_OUT:output_tensor"); - auto options = config.mutable_options()->MutableExtension( - VectorIntToTensorCalculatorOptions::ext); - options->set_input_size(input_size); - options->set_transpose(transpose); - options->set_tensor_data_type(tensor_data_type); - runner_ = ::absl::make_unique(config); - } - - void TestConvertFromVectoVectorInt(const bool transpose) { - SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_2D, - tensorflow::DT_INT32, transpose, false); - auto input = ::absl::make_unique>>( - 2, std::vector(2)); - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - input->at(i).at(j) = i * 2 + j; - } - } - - const int64 time = 1234; - runner_->MutableInputs() - ->Tag("VECTOR_INT") - .packets.push_back(Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Tag("TENSOR_OUT").packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(2, output_tensor.dims()); - EXPECT_EQ(tf::DT_INT32, output_tensor.dtype()); - const auto matrix = output_tensor.matrix(); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - if (!transpose) { - EXPECT_EQ(i * 2 + j, matrix(i, j)); - } else { - EXPECT_EQ(j * 2 + i, matrix(i, j)); - } - } - } - } - - std::unique_ptr runner_; -}; - -TEST_F(VectorIntToTensorCalculatorTest, TestSingleValue) { - SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D, - tensorflow::DT_INT32, false, true); - const int64 time = 1234; - runner_->MutableInputs() - ->Tag("SINGLE_INT") - .packets.push_back(MakePacket(1).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Tag("TENSOR_OUT").packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_INT32, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - EXPECT_EQ(1, vec(0)); -} - -TEST_F(VectorIntToTensorCalculatorTest, TesOneDim) { - SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D, - tensorflow::DT_INT32, false, false); - auto input = ::absl::make_unique>(5); - for (int i = 0; i < 5; ++i) { - input->at(i) = i; - } - const int64 time = 1234; - runner_->MutableInputs() - ->Tag("VECTOR_INT") - .packets.push_back(Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Tag("TENSOR_OUT").packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_INT32, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, vec(i)); - } -} - -TEST_F(VectorIntToTensorCalculatorTest, TestTwoDims) { - for (bool transpose : {false, true}) { - TestConvertFromVectoVectorInt(transpose); - } -} - -TEST_F(VectorIntToTensorCalculatorTest, TestInt64) { - SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D, - tensorflow::DT_INT64, false, true); - const int64 time = 1234; - runner_->MutableInputs() - ->Tag("SINGLE_INT") - .packets.push_back(MakePacket(1LL << 31).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Tag("TENSOR_OUT").packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_INT64, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - // 1LL << 31 overflows the positive int and becomes negative. - EXPECT_EQ(static_cast(1LL << 31), vec(0)); -} - -TEST_F(VectorIntToTensorCalculatorTest, TestUint8) { - SetUpRunner(VectorIntToTensorCalculatorOptions::INPUT_1D, - tensorflow::DT_UINT8, false, false); - auto input = ::absl::make_unique>(5); - for (int i = 0; i < 5; ++i) { - input->at(i) = i; - } - const int64 time = 1234; - runner_->MutableInputs() - ->Tag("VECTOR_INT") - .packets.push_back(Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Tag("TENSOR_OUT").packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_UINT8, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(i, vec(i)); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc deleted file mode 100644 index 0e579009b..000000000 --- a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Converts vector (or vector>) to 1D (or 2D) -// tf::Tensor. - -#include "mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.h" - -namespace mediapipe { - -namespace { -auto& INPUT_1D = VectorStringToTensorCalculatorOptions::INPUT_1D; -auto& INPUT_2D = VectorStringToTensorCalculatorOptions::INPUT_2D; -} // namespace - -namespace tf = ::tensorflow; - -// The calculator expects one input (a packet containing a vector -// or vector>) and generates one output (a packet containing -// a tf::Tensor containing the same data). The output tensor will be either 1D -// or 2D with dimensions corresponding to the input vector std::string. It will -// hold DT_STRING values. -// -// Example config: -// node { -// calculator: "VectorStringToTensorCalculator" -// input_stream: "vector_string_features" -// output_stream: "tensor_features" -// } -class VectorStringToTensorCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - VectorStringToTensorCalculatorOptions options_; -}; -REGISTER_CALCULATOR(VectorStringToTensorCalculator); - -absl::Status VectorStringToTensorCalculator::GetContract( - CalculatorContract* cc) { - const auto& options = cc->Options(); - // Start with only one input packet. - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1) - << "Only one input stream is supported."; - if (options.input_size() == INPUT_2D) { - cc->Inputs().Index(0).Set>>( - /* "Input vector>." */); - } else if (options.input_size() == INPUT_1D) { - cc->Inputs().Index(0).Set>( - // Input vector. - ); - } else { - LOG(FATAL) << "input size not supported"; - } - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) - << "Only one output stream is supported."; - cc->Outputs().Index(0).Set( - // Output stream with data as tf::Tensor and the same TimeSeriesHeader. - ); - return absl::OkStatus(); -} - -absl::Status VectorStringToTensorCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - cc->SetOffset(0); - return absl::OkStatus(); -} - -absl::Status VectorStringToTensorCalculator::Process(CalculatorContext* cc) { - tf::TensorShape tensor_shape; - if (options_.input_size() == INPUT_2D) { - const std::vector>& input = - cc->Inputs() - .Index(0) - .Value() - .Get>>(); - - const int32 rows = input.size(); - RET_CHECK_GE(rows, 1); - const int32 cols = input[0].size(); - RET_CHECK_GE(cols, 1); - for (int i = 1; i < rows; ++i) { - RET_CHECK_EQ(input[i].size(), cols); - } - if (options_.transpose()) { - tensor_shape = tf::TensorShape({cols, rows}); - } else { - tensor_shape = tf::TensorShape({rows, cols}); - } - auto output = ::absl::make_unique(tf::DT_STRING, tensor_shape); - for (int r = 0; r < rows; ++r) { - for (int c = 0; c < cols; ++c) { - if (options_.transpose()) { - output->tensor()(c, r) = input[r][c]; - } else { - output->tensor()(r, c) = input[r][c]; - } - } - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else if (options_.input_size() == INPUT_1D) { - const std::vector& input = - cc->Inputs().Index(0).Value().Get>(); - RET_CHECK_GE(input.size(), 1); - const int32 length = input.size(); - tensor_shape = tf::TensorShape({length}); - auto output = ::absl::make_unique(tf::DT_STRING, tensor_shape); - for (int i = 0; i < length; ++i) { - output->tensor()(i) = input.at(i); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - } else { - LOG(FATAL) << "input size not supported"; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.proto b/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.proto deleted file mode 100644 index 908d98dff..000000000 --- a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message VectorStringToTensorCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional VectorStringToTensorCalculatorOptions ext = 357221188; - } - enum InputSize { - UNKNOWN = 0; - INPUT_1D = 1; - INPUT_2D = 2; - } - - // If input_size is INPUT_2D, unpack a vector> to a - // 2d tensor (matrix). If INPUT_1D, - // convert a vector into a 1d tensor (vector). - optional InputSize input_size = 1 [default = INPUT_1D]; - - // If true, the output tensor is transposed. - // Otherwise, the output tensor is not transposed. - // It will be ignored if input_size is INPUT_1D. - optional bool transpose = 2 [default = false]; -} diff --git a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_test.cc b/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_test.cc deleted file mode 100644 index 5921bd1b0..000000000 --- a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_test.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2018 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/types.pb.h" - -namespace mediapipe { - -namespace { - -namespace tf = ::tensorflow; - -class VectorStringToTensorCalculatorTest : public ::testing::Test { - protected: - void SetUpRunner( - const VectorStringToTensorCalculatorOptions::InputSize input_size, - const bool transpose) { - CalculatorGraphConfig::Node config; - config.set_calculator("VectorStringToTensorCalculator"); - config.add_input_stream("input_string"); - config.add_output_stream("output_tensor"); - auto options = config.mutable_options()->MutableExtension( - VectorStringToTensorCalculatorOptions::ext); - options->set_input_size(input_size); - options->set_transpose(transpose); - runner_ = ::absl::make_unique(config); - } - - void TestConvertFromVectoVectorString(const bool transpose) { - SetUpRunner(VectorStringToTensorCalculatorOptions::INPUT_2D, transpose); - auto input = ::absl::make_unique>>( - 2, std::vector(2)); - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - input->at(i).at(j) = absl::StrCat(i, j); - } - } - - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(2, output_tensor.dims()); - EXPECT_EQ(tf::DT_STRING, output_tensor.dtype()); - const auto matrix = output_tensor.matrix(); - - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < 2; ++j) { - if (!transpose) { - EXPECT_EQ(absl::StrCat(i, j), matrix(i, j)); - } else { - EXPECT_EQ(absl::StrCat(j, i), matrix(i, j)); - } - } - } - } - - std::unique_ptr runner_; -}; - -TEST_F(VectorStringToTensorCalculatorTest, ConvertsFromVectorString) { - SetUpRunner(VectorStringToTensorCalculatorOptions::INPUT_1D, false); - auto input = ::absl::make_unique>(5); - for (int i = 0; i < 5; ++i) { - input->at(i) = absl::StrCat(i); - } - const int64 time = 1234; - runner_->MutableInputs()->Index(0).packets.push_back( - Adopt(input.release()).At(Timestamp(time))); - - EXPECT_TRUE(runner_->Run().ok()); - - const std::vector& output_packets = - runner_->Outputs().Index(0).packets; - EXPECT_EQ(1, output_packets.size()); - EXPECT_EQ(time, output_packets[0].Timestamp().Value()); - const tf::Tensor& output_tensor = output_packets[0].Get(); - - EXPECT_EQ(1, output_tensor.dims()); - EXPECT_EQ(tf::DT_STRING, output_tensor.dtype()); - const auto vec = output_tensor.vec(); - - for (int i = 0; i < 5; ++i) { - EXPECT_EQ(absl::StrCat(i), vec(i)); - } -} - -TEST_F(VectorStringToTensorCalculatorTest, ConvertsFromVectorVectorString) { - for (bool transpose : {false, true}) { - TestConvertFromVectoVectorString(transpose); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD deleted file mode 100644 index 2d1037d20..000000000 --- a/mediapipe/calculators/tflite/BUILD +++ /dev/null @@ -1,527 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") -load("@bazel_skylib//lib:selects.bzl", "selects") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -mediapipe_proto_library( - name = "ssd_anchors_calculator_proto", - srcs = ["ssd_anchors_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_custom_op_resolver_calculator_proto", - srcs = ["tflite_custom_op_resolver_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_inference_calculator_proto", - srcs = ["tflite_inference_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_converter_calculator_proto", - srcs = ["tflite_converter_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_tensors_to_segmentation_calculator_proto", - srcs = ["tflite_tensors_to_segmentation_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_tensors_to_detections_calculator_proto", - srcs = ["tflite_tensors_to_detections_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_tensors_to_classification_calculator_proto", - srcs = ["tflite_tensors_to_classification_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "tflite_tensors_to_landmarks_calculator_proto", - srcs = ["tflite_tensors_to_landmarks_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "ssd_anchors_calculator", - srcs = ["ssd_anchors_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":ssd_anchors_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats/object_detection:anchor_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "tflite_custom_op_resolver_calculator", - srcs = ["tflite_custom_op_resolver_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tflite_custom_op_resolver_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util/tflite:cpu_op_resolver", - "//mediapipe/util/tflite:op_resolver", - ], - alwayslink = 1, -) - -filegroup( - name = "anchor_golden_files", - srcs = [ - "testdata/anchor_golden_file_0.txt", - "testdata/anchor_golden_file_1.txt", - ], -) - -cc_test( - name = "ssd_anchors_calculator_test", - srcs = ["ssd_anchors_calculator_test.cc"], - data = [":anchor_golden_files"], - deps = [ - ":ssd_anchors_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats/object_detection:anchor_cc_proto", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/flags:flag", - ], -) - -selects.config_setting_group( - name = "gpu_inference_disabled", - match_any = [ - "//mediapipe/gpu:disable_gpu", - ], -) - -cc_library( - name = "tflite_inference_calculator", - srcs = ["tflite_inference_calculator.cc"], - copts = select({ - "//mediapipe:ios": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:ios": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":tflite_inference_calculator_cc_proto", - "@com_google_absl//absl/memory", - "//mediapipe/framework:calculator_framework", - "//mediapipe/util/tflite:config", - "//mediapipe/util/tflite:tflite_model_loader", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/port:ret_check", - ] + selects.with_or({ - ":gpu_inference_disabled": [], - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/objc:mediapipe_framework_ios", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal", - ], - "//conditions:default": [ - "//mediapipe/util/tflite:tflite_gpu_runner", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape", - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - ], - }) + select({ - "//conditions:default": [], - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - "@org_tensorflow//tensorflow/lite/delegates/nnapi:nnapi_delegate", - ], - }) + select({ - "//conditions:default": [ - "//mediapipe/util:cpu_util", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tflite_converter_calculator", - srcs = ["tflite_converter_calculator.cc"], - copts = select({ - "//mediapipe:ios": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:ios": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/util/tflite:config", - ":tflite_converter_calculator_cc_proto", - "//mediapipe/util:resource_util", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/port:ret_check", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - ] + selects.with_or({ - ":gpu_inference_disabled": [], - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/objc:mediapipe_framework_ios", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", - ], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - ], - }) + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gpu_buffer", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tflite_model_calculator", - srcs = ["tflite_model_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework/port:ret_check", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "tflite_tensors_to_segmentation_calculator", - srcs = ["tflite_tensors_to_segmentation_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tflite_tensors_to_segmentation_calculator_cc_proto", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/util:resource_util", - "@org_tensorflow//tensorflow/lite:framework", - ] + selects.with_or({ - ":gpu_inference_disabled": [], - "//mediapipe:ios": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:shader_util", - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_texture", - ], - }), - alwayslink = 1, -) - -cc_test( - name = "tflite_tensors_to_classification_calculator_test", - srcs = ["tflite_tensors_to_classification_calculator_test.cc"], - data = ["testdata/labelmap.txt"], - deps = [ - ":tflite_tensors_to_classification_calculator", - ":tflite_tensors_to_classification_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/memory", - "@com_google_googletest//:gtest_main", - "@org_tensorflow//tensorflow/lite:framework", - ], -) - -cc_library( - name = "tflite_tensors_to_detections_calculator", - srcs = ["tflite_tensors_to_detections_calculator.cc"], - copts = select({ - "//mediapipe:ios": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - linkopts = select({ - "//mediapipe:ios": [ - "-framework CoreVideo", - "-framework MetalKit", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/util/tflite:config", - ":tflite_tensors_to_detections_calculator_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/formats/object_detection:anchor_cc_proto", - "//mediapipe/framework/port:ret_check", - "@org_tensorflow//tensorflow/lite:framework", - ] + selects.with_or({ - ":gpu_inference_disabled": [], - "//mediapipe:ios": [ - "//mediapipe/gpu:MPPMetalUtil", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:MPPMetalHelper", - "//mediapipe/objc:mediapipe_framework_ios", - "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", - ], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_program", - "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_shader", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tflite_tensors_to_classification_calculator", - srcs = ["tflite_tensors_to_classification_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tflite_tensors_to_classification_calculator_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/types:span", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:resource_util", - "@org_tensorflow//tensorflow/lite:framework", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tflite_tensors_to_landmarks_calculator", - srcs = ["tflite_tensors_to_landmarks_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tflite_tensors_to_landmarks_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -cc_library( - name = "tflite_tensors_to_floats_calculator", - srcs = ["tflite_tensors_to_floats_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "@org_tensorflow//tensorflow/lite:framework", - ], - alwayslink = 1, -) - -# To run this with native GPU on Linux, use: -# bazel test //mediapipe/calculators/tflite:tflite_inference_calculator_test --copt=-DTFLITE_GPU_EXTRA_GLES_DEPS --copt=-DMESA_EGL_NO_X11_HEADERS --copt=-DEGL_NO_X11 --config=grte_v5 --test_strategy=local -cc_test( - name = "tflite_inference_calculator_test", - srcs = [ - "tflite_inference_calculator_test.cc", - "tflite_inference_calculator_test_common.h", - ], - data = ["testdata/add.bin"], - linkstatic = 1, - deps = [ - ":tflite_inference_calculator", - ":tflite_inference_calculator_cc_proto", - ":tflite_model_calculator", - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/util:local_file_contents_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:optional", - "@org_tensorflow//tensorflow/lite:framework", - "@org_tensorflow//tensorflow/lite:type_to_tflitetype", - "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", - "@org_tensorflow//tensorflow/lite/kernels/internal:tensor", - ], -) - -cc_test( - name = "tflite_converter_calculator_test", - srcs = ["tflite_converter_calculator_test.cc"], - deps = [ - ":tflite_converter_calculator", - ":tflite_converter_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/lite:framework", - ], -) - -cc_test( - name = "tflite_model_calculator_test", - srcs = ["tflite_model_calculator_test.cc"], - data = ["testdata/add.bin"], - deps = [ - ":tflite_model_calculator", - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/util:local_file_contents_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@org_tensorflow//tensorflow/lite:framework", - ], -) diff --git a/mediapipe/calculators/tflite/ssd_anchors_calculator.cc b/mediapipe/calculators/tflite/ssd_anchors_calculator.cc deleted file mode 100644 index f618b2f6a..000000000 --- a/mediapipe/calculators/tflite/ssd_anchors_calculator.cc +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/calculators/tflite/ssd_anchors_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/object_detection/anchor.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -float CalculateScale(float min_scale, float max_scale, int stride_index, - int num_strides) { - if (num_strides == 1) { - return (min_scale + max_scale) * 0.5f; - } else { - return min_scale + - (max_scale - min_scale) * 1.0 * stride_index / (num_strides - 1.0f); - } -} - -} // namespace - -// Generate anchors for SSD object detection model. -// Output: -// ANCHORS: A list of anchors. Model generates predictions based on the -// offsets of these anchors. -// -// Usage example: -// node { -// calculator: "SsdAnchorsCalculator" -// output_side_packet: "anchors" -// options { -// [mediapipe.SsdAnchorsCalculatorOptions.ext] { -// num_layers: 6 -// min_scale: 0.2 -// max_scale: 0.95 -// input_size_height: 300 -// input_size_width: 300 -// anchor_offset_x: 0.5 -// anchor_offset_y: 0.5 -// strides: 16 -// strides: 32 -// strides: 64 -// strides: 128 -// strides: 256 -// strides: 512 -// aspect_ratios: 1.0 -// aspect_ratios: 2.0 -// aspect_ratios: 0.5 -// aspect_ratios: 3.0 -// aspect_ratios: 0.3333 -// reduce_boxes_in_lowest_layer: true -// } -// } -// } -class SsdAnchorsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->OutputSidePackets().Index(0).Set>(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - const SsdAnchorsCalculatorOptions& options = - cc->Options(); - - auto anchors = absl::make_unique>(); - MP_RETURN_IF_ERROR(GenerateAnchors(anchors.get(), options)); - cc->OutputSidePackets().Index(0).Set(Adopt(anchors.release())); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } - - private: - static absl::Status GenerateAnchors( - std::vector* anchors, const SsdAnchorsCalculatorOptions& options); -}; -REGISTER_CALCULATOR(SsdAnchorsCalculator); - -absl::Status SsdAnchorsCalculator::GenerateAnchors( - std::vector* anchors, const SsdAnchorsCalculatorOptions& options) { - // Verify the options. - if (!options.feature_map_height_size() && !options.strides_size()) { - return absl::InvalidArgumentError( - "Both feature map shape and strides are missing. Must provide either " - "one."); - } - if (options.feature_map_height_size()) { - if (options.strides_size()) { - LOG(ERROR) << "Found feature map shapes. Strides will be ignored."; - } - CHECK_EQ(options.feature_map_height_size(), options.num_layers()); - CHECK_EQ(options.feature_map_height_size(), - options.feature_map_width_size()); - } else { - CHECK_EQ(options.strides_size(), options.num_layers()); - } - - int layer_id = 0; - while (layer_id < options.num_layers()) { - std::vector anchor_height; - std::vector anchor_width; - std::vector aspect_ratios; - std::vector scales; - - // For same strides, we merge the anchors in the same order. - int last_same_stride_layer = layer_id; - while (last_same_stride_layer < options.strides_size() && - options.strides(last_same_stride_layer) == - options.strides(layer_id)) { - const float scale = - CalculateScale(options.min_scale(), options.max_scale(), - last_same_stride_layer, options.strides_size()); - if (last_same_stride_layer == 0 && - options.reduce_boxes_in_lowest_layer()) { - // For first layer, it can be specified to use predefined anchors. - aspect_ratios.push_back(1.0); - aspect_ratios.push_back(2.0); - aspect_ratios.push_back(0.5); - scales.push_back(0.1); - scales.push_back(scale); - scales.push_back(scale); - } else { - for (int aspect_ratio_id = 0; - aspect_ratio_id < options.aspect_ratios_size(); - ++aspect_ratio_id) { - aspect_ratios.push_back(options.aspect_ratios(aspect_ratio_id)); - scales.push_back(scale); - } - if (options.interpolated_scale_aspect_ratio() > 0.0) { - const float scale_next = - last_same_stride_layer == options.strides_size() - 1 - ? 1.0f - : CalculateScale(options.min_scale(), options.max_scale(), - last_same_stride_layer + 1, - options.strides_size()); - scales.push_back(std::sqrt(scale * scale_next)); - aspect_ratios.push_back(options.interpolated_scale_aspect_ratio()); - } - } - last_same_stride_layer++; - } - - for (int i = 0; i < aspect_ratios.size(); ++i) { - const float ratio_sqrts = std::sqrt(aspect_ratios[i]); - anchor_height.push_back(scales[i] / ratio_sqrts); - anchor_width.push_back(scales[i] * ratio_sqrts); - } - - int feature_map_height = 0; - int feature_map_width = 0; - if (options.feature_map_height_size()) { - feature_map_height = options.feature_map_height(layer_id); - feature_map_width = options.feature_map_width(layer_id); - } else { - const int stride = options.strides(layer_id); - feature_map_height = - std::ceil(1.0f * options.input_size_height() / stride); - feature_map_width = std::ceil(1.0f * options.input_size_width() / stride); - } - - for (int y = 0; y < feature_map_height; ++y) { - for (int x = 0; x < feature_map_width; ++x) { - for (int anchor_id = 0; anchor_id < anchor_height.size(); ++anchor_id) { - // TODO: Support specifying anchor_offset_x, anchor_offset_y. - const float x_center = - (x + options.anchor_offset_x()) * 1.0f / feature_map_width; - const float y_center = - (y + options.anchor_offset_y()) * 1.0f / feature_map_height; - - Anchor new_anchor; - new_anchor.set_x_center(x_center); - new_anchor.set_y_center(y_center); - - if (options.fixed_anchor_size()) { - new_anchor.set_w(1.0f); - new_anchor.set_h(1.0f); - } else { - new_anchor.set_w(anchor_width[anchor_id]); - new_anchor.set_h(anchor_height[anchor_id]); - } - anchors->push_back(new_anchor); - } - } - } - layer_id = last_same_stride_layer; - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/ssd_anchors_calculator.proto b/mediapipe/calculators/tflite/ssd_anchors_calculator.proto deleted file mode 100644 index c89248822..000000000 --- a/mediapipe/calculators/tflite/ssd_anchors_calculator.proto +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Options to generate anchors for SSD object detection models. -message SsdAnchorsCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional SsdAnchorsCalculatorOptions ext = 247258239; - } - // Size of input images. - required int32 input_size_width = 1; - required int32 input_size_height = 2; - - // Min and max scales for generating anchor boxes on feature maps. - required float min_scale = 3; - required float max_scale = 4; - - // The offset for the center of anchors. The value is in the scale of stride. - // E.g. 0.5 meaning 0.5 * |current_stride| in pixels. - required float anchor_offset_x = 5 [default = 0.5]; - required float anchor_offset_y = 6 [default = 0.5]; - - // Number of output feature maps to generate the anchors on. - required int32 num_layers = 7; - // Sizes of output feature maps to create anchors. Either feature_map size or - // stride should be provided. - repeated int32 feature_map_width = 8; - repeated int32 feature_map_height = 9; - // Strides of each output feature maps. - repeated int32 strides = 10; - - // List of different aspect ratio to generate anchors. - repeated float aspect_ratios = 11; - - // A boolean to indicate whether the fixed 3 boxes per location is used in the - // lowest layer. - optional bool reduce_boxes_in_lowest_layer = 12 [default = false]; - // An additional anchor is added with this aspect ratio and a scale - // interpolated between the scale for a layer and the scale for the next layer - // (1.0 for the last layer). This anchor is not included if this value is 0. - optional float interpolated_scale_aspect_ratio = 13 [default = 1.0]; - - // Whether use fixed width and height (e.g. both 1.0f) for each anchor. - // This option can be used when the predicted anchor width and height are in - // pixels. - optional bool fixed_anchor_size = 14 [default = false]; -} diff --git a/mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc b/mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc deleted file mode 100644 index 3b72a287e..000000000 --- a/mediapipe/calculators/tflite/ssd_anchors_calculator_test.cc +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/object_detection/anchor.pb.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -std::string GetGoldenFilePath(const std::string& filename) { - return mediapipe::file::JoinPath( - "./", "mediapipe/calculators/tflite/testdata/" + filename); -} - -void ParseAnchorsFromText(const std::string& text, - std::vector* anchors) { - const std::string line_delimiter = "\n"; - const std::string number_delimiter = ","; - - std::istringstream stream(text); - std::string line; - while (std::getline(stream, line)) { - Anchor anchor; - float values[4]; - std::string::size_type pos; - for (int i = 0; i < 4; ++i) { - values[i] = std::stof(line, &pos); - line = line.substr(pos); - } - anchor.set_x_center(values[0]); - anchor.set_y_center(values[1]); - anchor.set_w(values[2]); - anchor.set_h(values[3]); - anchors->push_back(anchor); - } -} - -void CompareAnchors(const std::vector& anchors_0, - const std::vector& anchors_1) { - EXPECT_EQ(anchors_0.size(), anchors_1.size()); - for (int i = 0; i < anchors_0.size(); ++i) { - const auto& anchor_0 = anchors_0[i]; - const auto& anchor_1 = anchors_1[i]; - EXPECT_THAT(anchor_0.x_center(), - testing::FloatNear(anchor_1.x_center(), 1e-5)); - EXPECT_THAT(anchor_0.y_center(), - testing::FloatNear(anchor_1.y_center(), 1e-5)); - } -} - -TEST(SsdAnchorCalculatorTest, FaceDetectionConfig) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 5 - min_scale: 0.1171875 - max_scale: 0.75 - input_size_height: 256 - input_size_width: 256 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 8 - strides: 16 - strides: 32 - strides: 32 - strides: 32 - aspect_ratios: 1.0 - fixed_anchor_size: true - } - } - )pb")); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - - const auto& anchors = - runner.OutputSidePackets().Index(0).Get>(); - std::string anchors_string; - MP_EXPECT_OK(mediapipe::file::GetContents( - GetGoldenFilePath("anchor_golden_file_0.txt"), &anchors_string)); - - std::vector anchors_golden; - ParseAnchorsFromText(anchors_string, &anchors_golden); - - CompareAnchors(anchors, anchors_golden); -} - -TEST(SsdAnchorCalculatorTest, MobileSSDConfig) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 300 - input_size_width: 300 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } - )pb")); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const auto& anchors = - runner.OutputSidePackets().Index(0).Get>(); - - std::string anchors_string; - MP_EXPECT_OK(mediapipe::file::GetContents( - GetGoldenFilePath("anchor_golden_file_1.txt"), &anchors_string)); - - std::vector anchors_golden; - ParseAnchorsFromText(anchors_string, &anchors_golden); - - CompareAnchors(anchors, anchors_golden); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/testdata/README.md b/mediapipe/calculators/tflite/testdata/README.md deleted file mode 100644 index c0efdcf07..000000000 --- a/mediapipe/calculators/tflite/testdata/README.md +++ /dev/null @@ -1,2 +0,0 @@ -The model files add.bin, add_quantized.bin -(and corresponding metatada json files) come from tensorflow/lite/testdata/ diff --git a/mediapipe/calculators/tflite/testdata/add.bin b/mediapipe/calculators/tflite/testdata/add.bin deleted file mode 100644 index b4c02350c..000000000 Binary files a/mediapipe/calculators/tflite/testdata/add.bin and /dev/null differ diff --git a/mediapipe/calculators/tflite/testdata/add.json b/mediapipe/calculators/tflite/testdata/add.json deleted file mode 100644 index f589bebfb..000000000 --- a/mediapipe/calculators/tflite/testdata/add.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - version: 3, - operator_codes: [ - { - } - ], - subgraphs: [ - { - tensors: [ - { - shape: [ - 1, - 8, - 8, - 3 - ], - name: "add" - }, - { - shape: [ - 1, - 8, - 8, - 3 - ], - name: "input" - }, - { - shape: [ - 1, - 8, - 8, - 3 - ], - name: "output" - } - ], - inputs: [ - 1 - ], - outputs: [ - 2 - ], - operators: [ - { - inputs: [ - 1, - 1 - ], - outputs: [ - 0 - ], - builtin_options_type: "AddOptions", - builtin_options: { - } - }, - { - inputs: [ - 0, - 1 - ], - outputs: [ - 2 - ], - builtin_options_type: "AddOptions", - builtin_options: { - } - } - ] - } - ], - buffers: [ - { - data: [ - - ] - } - ] -} diff --git a/mediapipe/calculators/tflite/testdata/add_quantized.bin b/mediapipe/calculators/tflite/testdata/add_quantized.bin deleted file mode 100644 index 07d48b93e..000000000 Binary files a/mediapipe/calculators/tflite/testdata/add_quantized.bin and /dev/null differ diff --git a/mediapipe/calculators/tflite/testdata/add_quantized.json b/mediapipe/calculators/tflite/testdata/add_quantized.json deleted file mode 100644 index f70ed8143..000000000 --- a/mediapipe/calculators/tflite/testdata/add_quantized.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - version: 3, - operator_codes: [ - { - } - ], - subgraphs: [ - { - tensors: [ - { - shape: [ - 1, - 8, - 8, - 3 - ], - name: "add", - quantization: { - min: [ - 0.0 - ], - max: [ - 1.0 - ], - scale: [ - 0.003922 - ], - zero_point: [ - 0 - ] - } - }, - { - shape: [ - 1, - 8, - 8, - 3 - ], - type: "UINT8", - name: "input", - quantization: { - min: [ - 0.0 - ], - max: [ - 1.0 - ], - scale: [ - 0.003922 - ], - zero_point: [ - 0 - ] - } - }, - { - shape: [ - 1, - 8, - 8, - 3 - ], - type: "UINT8", - name: "output", - quantization: { - min: [ - 0.0 - ], - max: [ - 1.0 - ], - scale: [ - 0.003922 - ], - zero_point: [ - 0 - ] - } - } - ], - inputs: [ - 1 - ], - outputs: [ - 2 - ], - operators: [ - { - inputs: [ - 1, - 1 - ], - outputs: [ - 0 - ], - builtin_options_type: "AddOptions", - builtin_options: { - } - }, - { - inputs: [ - 0, - 1 - ], - outputs: [ - 2 - ], - builtin_options_type: "AddOptions", - builtin_options: { - } - } - ] - } - ], - buffers: [ - { - data: [ - - ] - } - ] -} diff --git a/mediapipe/calculators/tflite/testdata/anchor_golden_file_0.txt b/mediapipe/calculators/tflite/testdata/anchor_golden_file_0.txt deleted file mode 100644 index 3d936c86b..000000000 --- a/mediapipe/calculators/tflite/testdata/anchor_golden_file_0.txt +++ /dev/null @@ -1,2944 +0,0 @@ -0.015625 0.015625 1 1 -0.015625 0.015625 1 1 -0.046875 0.015625 1 1 -0.046875 0.015625 1 1 -0.078125 0.015625 1 1 -0.078125 0.015625 1 1 -0.109375 0.015625 1 1 -0.109375 0.015625 1 1 -0.140625 0.015625 1 1 -0.140625 0.015625 1 1 -0.171875 0.015625 1 1 -0.171875 0.015625 1 1 -0.203125 0.015625 1 1 -0.203125 0.015625 1 1 -0.234375 0.015625 1 1 -0.234375 0.015625 1 1 -0.265625 0.015625 1 1 -0.265625 0.015625 1 1 -0.296875 0.015625 1 1 -0.296875 0.015625 1 1 -0.328125 0.015625 1 1 -0.328125 0.015625 1 1 -0.359375 0.015625 1 1 -0.359375 0.015625 1 1 -0.390625 0.015625 1 1 -0.390625 0.015625 1 1 -0.421875 0.015625 1 1 -0.421875 0.015625 1 1 -0.453125 0.015625 1 1 -0.453125 0.015625 1 1 -0.484375 0.015625 1 1 -0.484375 0.015625 1 1 -0.515625 0.015625 1 1 -0.515625 0.015625 1 1 -0.546875 0.015625 1 1 -0.546875 0.015625 1 1 -0.578125 0.015625 1 1 -0.578125 0.015625 1 1 -0.609375 0.015625 1 1 -0.609375 0.015625 1 1 -0.640625 0.015625 1 1 -0.640625 0.015625 1 1 -0.671875 0.015625 1 1 -0.671875 0.015625 1 1 -0.703125 0.015625 1 1 -0.703125 0.015625 1 1 -0.734375 0.015625 1 1 -0.734375 0.015625 1 1 -0.765625 0.015625 1 1 -0.765625 0.015625 1 1 -0.796875 0.015625 1 1 -0.796875 0.015625 1 1 -0.828125 0.015625 1 1 -0.828125 0.015625 1 1 -0.859375 0.015625 1 1 -0.859375 0.015625 1 1 -0.890625 0.015625 1 1 -0.890625 0.015625 1 1 -0.921875 0.015625 1 1 -0.921875 0.015625 1 1 -0.953125 0.015625 1 1 -0.953125 0.015625 1 1 -0.984375 0.015625 1 1 -0.984375 0.015625 1 1 -0.015625 0.046875 1 1 -0.015625 0.046875 1 1 -0.046875 0.046875 1 1 -0.046875 0.046875 1 1 -0.078125 0.046875 1 1 -0.078125 0.046875 1 1 -0.109375 0.046875 1 1 -0.109375 0.046875 1 1 -0.140625 0.046875 1 1 -0.140625 0.046875 1 1 -0.171875 0.046875 1 1 -0.171875 0.046875 1 1 -0.203125 0.046875 1 1 -0.203125 0.046875 1 1 -0.234375 0.046875 1 1 -0.234375 0.046875 1 1 -0.265625 0.046875 1 1 -0.265625 0.046875 1 1 -0.296875 0.046875 1 1 -0.296875 0.046875 1 1 -0.328125 0.046875 1 1 -0.328125 0.046875 1 1 -0.359375 0.046875 1 1 -0.359375 0.046875 1 1 -0.390625 0.046875 1 1 -0.390625 0.046875 1 1 -0.421875 0.046875 1 1 -0.421875 0.046875 1 1 -0.453125 0.046875 1 1 -0.453125 0.046875 1 1 -0.484375 0.046875 1 1 -0.484375 0.046875 1 1 -0.515625 0.046875 1 1 -0.515625 0.046875 1 1 -0.546875 0.046875 1 1 -0.546875 0.046875 1 1 -0.578125 0.046875 1 1 -0.578125 0.046875 1 1 -0.609375 0.046875 1 1 -0.609375 0.046875 1 1 -0.640625 0.046875 1 1 -0.640625 0.046875 1 1 -0.671875 0.046875 1 1 -0.671875 0.046875 1 1 -0.703125 0.046875 1 1 -0.703125 0.046875 1 1 -0.734375 0.046875 1 1 -0.734375 0.046875 1 1 -0.765625 0.046875 1 1 -0.765625 0.046875 1 1 -0.796875 0.046875 1 1 -0.796875 0.046875 1 1 -0.828125 0.046875 1 1 -0.828125 0.046875 1 1 -0.859375 0.046875 1 1 -0.859375 0.046875 1 1 -0.890625 0.046875 1 1 -0.890625 0.046875 1 1 -0.921875 0.046875 1 1 -0.921875 0.046875 1 1 -0.953125 0.046875 1 1 -0.953125 0.046875 1 1 -0.984375 0.046875 1 1 -0.984375 0.046875 1 1 -0.015625 0.078125 1 1 -0.015625 0.078125 1 1 -0.046875 0.078125 1 1 -0.046875 0.078125 1 1 -0.078125 0.078125 1 1 -0.078125 0.078125 1 1 -0.109375 0.078125 1 1 -0.109375 0.078125 1 1 -0.140625 0.078125 1 1 -0.140625 0.078125 1 1 -0.171875 0.078125 1 1 -0.171875 0.078125 1 1 -0.203125 0.078125 1 1 -0.203125 0.078125 1 1 -0.234375 0.078125 1 1 -0.234375 0.078125 1 1 -0.265625 0.078125 1 1 -0.265625 0.078125 1 1 -0.296875 0.078125 1 1 -0.296875 0.078125 1 1 -0.328125 0.078125 1 1 -0.328125 0.078125 1 1 -0.359375 0.078125 1 1 -0.359375 0.078125 1 1 -0.390625 0.078125 1 1 -0.390625 0.078125 1 1 -0.421875 0.078125 1 1 -0.421875 0.078125 1 1 -0.453125 0.078125 1 1 -0.453125 0.078125 1 1 -0.484375 0.078125 1 1 -0.484375 0.078125 1 1 -0.515625 0.078125 1 1 -0.515625 0.078125 1 1 -0.546875 0.078125 1 1 -0.546875 0.078125 1 1 -0.578125 0.078125 1 1 -0.578125 0.078125 1 1 -0.609375 0.078125 1 1 -0.609375 0.078125 1 1 -0.640625 0.078125 1 1 -0.640625 0.078125 1 1 -0.671875 0.078125 1 1 -0.671875 0.078125 1 1 -0.703125 0.078125 1 1 -0.703125 0.078125 1 1 -0.734375 0.078125 1 1 -0.734375 0.078125 1 1 -0.765625 0.078125 1 1 -0.765625 0.078125 1 1 -0.796875 0.078125 1 1 -0.796875 0.078125 1 1 -0.828125 0.078125 1 1 -0.828125 0.078125 1 1 -0.859375 0.078125 1 1 -0.859375 0.078125 1 1 -0.890625 0.078125 1 1 -0.890625 0.078125 1 1 -0.921875 0.078125 1 1 -0.921875 0.078125 1 1 -0.953125 0.078125 1 1 -0.953125 0.078125 1 1 -0.984375 0.078125 1 1 -0.984375 0.078125 1 1 -0.015625 0.109375 1 1 -0.015625 0.109375 1 1 -0.046875 0.109375 1 1 -0.046875 0.109375 1 1 -0.078125 0.109375 1 1 -0.078125 0.109375 1 1 -0.109375 0.109375 1 1 -0.109375 0.109375 1 1 -0.140625 0.109375 1 1 -0.140625 0.109375 1 1 -0.171875 0.109375 1 1 -0.171875 0.109375 1 1 -0.203125 0.109375 1 1 -0.203125 0.109375 1 1 -0.234375 0.109375 1 1 -0.234375 0.109375 1 1 -0.265625 0.109375 1 1 -0.265625 0.109375 1 1 -0.296875 0.109375 1 1 -0.296875 0.109375 1 1 -0.328125 0.109375 1 1 -0.328125 0.109375 1 1 -0.359375 0.109375 1 1 -0.359375 0.109375 1 1 -0.390625 0.109375 1 1 -0.390625 0.109375 1 1 -0.421875 0.109375 1 1 -0.421875 0.109375 1 1 -0.453125 0.109375 1 1 -0.453125 0.109375 1 1 -0.484375 0.109375 1 1 -0.484375 0.109375 1 1 -0.515625 0.109375 1 1 -0.515625 0.109375 1 1 -0.546875 0.109375 1 1 -0.546875 0.109375 1 1 -0.578125 0.109375 1 1 -0.578125 0.109375 1 1 -0.609375 0.109375 1 1 -0.609375 0.109375 1 1 -0.640625 0.109375 1 1 -0.640625 0.109375 1 1 -0.671875 0.109375 1 1 -0.671875 0.109375 1 1 -0.703125 0.109375 1 1 -0.703125 0.109375 1 1 -0.734375 0.109375 1 1 -0.734375 0.109375 1 1 -0.765625 0.109375 1 1 -0.765625 0.109375 1 1 -0.796875 0.109375 1 1 -0.796875 0.109375 1 1 -0.828125 0.109375 1 1 -0.828125 0.109375 1 1 -0.859375 0.109375 1 1 -0.859375 0.109375 1 1 -0.890625 0.109375 1 1 -0.890625 0.109375 1 1 -0.921875 0.109375 1 1 -0.921875 0.109375 1 1 -0.953125 0.109375 1 1 -0.953125 0.109375 1 1 -0.984375 0.109375 1 1 -0.984375 0.109375 1 1 -0.015625 0.140625 1 1 -0.015625 0.140625 1 1 -0.046875 0.140625 1 1 -0.046875 0.140625 1 1 -0.078125 0.140625 1 1 -0.078125 0.140625 1 1 -0.109375 0.140625 1 1 -0.109375 0.140625 1 1 -0.140625 0.140625 1 1 -0.140625 0.140625 1 1 -0.171875 0.140625 1 1 -0.171875 0.140625 1 1 -0.203125 0.140625 1 1 -0.203125 0.140625 1 1 -0.234375 0.140625 1 1 -0.234375 0.140625 1 1 -0.265625 0.140625 1 1 -0.265625 0.140625 1 1 -0.296875 0.140625 1 1 -0.296875 0.140625 1 1 -0.328125 0.140625 1 1 -0.328125 0.140625 1 1 -0.359375 0.140625 1 1 -0.359375 0.140625 1 1 -0.390625 0.140625 1 1 -0.390625 0.140625 1 1 -0.421875 0.140625 1 1 -0.421875 0.140625 1 1 -0.453125 0.140625 1 1 -0.453125 0.140625 1 1 -0.484375 0.140625 1 1 -0.484375 0.140625 1 1 -0.515625 0.140625 1 1 -0.515625 0.140625 1 1 -0.546875 0.140625 1 1 -0.546875 0.140625 1 1 -0.578125 0.140625 1 1 -0.578125 0.140625 1 1 -0.609375 0.140625 1 1 -0.609375 0.140625 1 1 -0.640625 0.140625 1 1 -0.640625 0.140625 1 1 -0.671875 0.140625 1 1 -0.671875 0.140625 1 1 -0.703125 0.140625 1 1 -0.703125 0.140625 1 1 -0.734375 0.140625 1 1 -0.734375 0.140625 1 1 -0.765625 0.140625 1 1 -0.765625 0.140625 1 1 -0.796875 0.140625 1 1 -0.796875 0.140625 1 1 -0.828125 0.140625 1 1 -0.828125 0.140625 1 1 -0.859375 0.140625 1 1 -0.859375 0.140625 1 1 -0.890625 0.140625 1 1 -0.890625 0.140625 1 1 -0.921875 0.140625 1 1 -0.921875 0.140625 1 1 -0.953125 0.140625 1 1 -0.953125 0.140625 1 1 -0.984375 0.140625 1 1 -0.984375 0.140625 1 1 -0.015625 0.171875 1 1 -0.015625 0.171875 1 1 -0.046875 0.171875 1 1 -0.046875 0.171875 1 1 -0.078125 0.171875 1 1 -0.078125 0.171875 1 1 -0.109375 0.171875 1 1 -0.109375 0.171875 1 1 -0.140625 0.171875 1 1 -0.140625 0.171875 1 1 -0.171875 0.171875 1 1 -0.171875 0.171875 1 1 -0.203125 0.171875 1 1 -0.203125 0.171875 1 1 -0.234375 0.171875 1 1 -0.234375 0.171875 1 1 -0.265625 0.171875 1 1 -0.265625 0.171875 1 1 -0.296875 0.171875 1 1 -0.296875 0.171875 1 1 -0.328125 0.171875 1 1 -0.328125 0.171875 1 1 -0.359375 0.171875 1 1 -0.359375 0.171875 1 1 -0.390625 0.171875 1 1 -0.390625 0.171875 1 1 -0.421875 0.171875 1 1 -0.421875 0.171875 1 1 -0.453125 0.171875 1 1 -0.453125 0.171875 1 1 -0.484375 0.171875 1 1 -0.484375 0.171875 1 1 -0.515625 0.171875 1 1 -0.515625 0.171875 1 1 -0.546875 0.171875 1 1 -0.546875 0.171875 1 1 -0.578125 0.171875 1 1 -0.578125 0.171875 1 1 -0.609375 0.171875 1 1 -0.609375 0.171875 1 1 -0.640625 0.171875 1 1 -0.640625 0.171875 1 1 -0.671875 0.171875 1 1 -0.671875 0.171875 1 1 -0.703125 0.171875 1 1 -0.703125 0.171875 1 1 -0.734375 0.171875 1 1 -0.734375 0.171875 1 1 -0.765625 0.171875 1 1 -0.765625 0.171875 1 1 -0.796875 0.171875 1 1 -0.796875 0.171875 1 1 -0.828125 0.171875 1 1 -0.828125 0.171875 1 1 -0.859375 0.171875 1 1 -0.859375 0.171875 1 1 -0.890625 0.171875 1 1 -0.890625 0.171875 1 1 -0.921875 0.171875 1 1 -0.921875 0.171875 1 1 -0.953125 0.171875 1 1 -0.953125 0.171875 1 1 -0.984375 0.171875 1 1 -0.984375 0.171875 1 1 -0.015625 0.203125 1 1 -0.015625 0.203125 1 1 -0.046875 0.203125 1 1 -0.046875 0.203125 1 1 -0.078125 0.203125 1 1 -0.078125 0.203125 1 1 -0.109375 0.203125 1 1 -0.109375 0.203125 1 1 -0.140625 0.203125 1 1 -0.140625 0.203125 1 1 -0.171875 0.203125 1 1 -0.171875 0.203125 1 1 -0.203125 0.203125 1 1 -0.203125 0.203125 1 1 -0.234375 0.203125 1 1 -0.234375 0.203125 1 1 -0.265625 0.203125 1 1 -0.265625 0.203125 1 1 -0.296875 0.203125 1 1 -0.296875 0.203125 1 1 -0.328125 0.203125 1 1 -0.328125 0.203125 1 1 -0.359375 0.203125 1 1 -0.359375 0.203125 1 1 -0.390625 0.203125 1 1 -0.390625 0.203125 1 1 -0.421875 0.203125 1 1 -0.421875 0.203125 1 1 -0.453125 0.203125 1 1 -0.453125 0.203125 1 1 -0.484375 0.203125 1 1 -0.484375 0.203125 1 1 -0.515625 0.203125 1 1 -0.515625 0.203125 1 1 -0.546875 0.203125 1 1 -0.546875 0.203125 1 1 -0.578125 0.203125 1 1 -0.578125 0.203125 1 1 -0.609375 0.203125 1 1 -0.609375 0.203125 1 1 -0.640625 0.203125 1 1 -0.640625 0.203125 1 1 -0.671875 0.203125 1 1 -0.671875 0.203125 1 1 -0.703125 0.203125 1 1 -0.703125 0.203125 1 1 -0.734375 0.203125 1 1 -0.734375 0.203125 1 1 -0.765625 0.203125 1 1 -0.765625 0.203125 1 1 -0.796875 0.203125 1 1 -0.796875 0.203125 1 1 -0.828125 0.203125 1 1 -0.828125 0.203125 1 1 -0.859375 0.203125 1 1 -0.859375 0.203125 1 1 -0.890625 0.203125 1 1 -0.890625 0.203125 1 1 -0.921875 0.203125 1 1 -0.921875 0.203125 1 1 -0.953125 0.203125 1 1 -0.953125 0.203125 1 1 -0.984375 0.203125 1 1 -0.984375 0.203125 1 1 -0.015625 0.234375 1 1 -0.015625 0.234375 1 1 -0.046875 0.234375 1 1 -0.046875 0.234375 1 1 -0.078125 0.234375 1 1 -0.078125 0.234375 1 1 -0.109375 0.234375 1 1 -0.109375 0.234375 1 1 -0.140625 0.234375 1 1 -0.140625 0.234375 1 1 -0.171875 0.234375 1 1 -0.171875 0.234375 1 1 -0.203125 0.234375 1 1 -0.203125 0.234375 1 1 -0.234375 0.234375 1 1 -0.234375 0.234375 1 1 -0.265625 0.234375 1 1 -0.265625 0.234375 1 1 -0.296875 0.234375 1 1 -0.296875 0.234375 1 1 -0.328125 0.234375 1 1 -0.328125 0.234375 1 1 -0.359375 0.234375 1 1 -0.359375 0.234375 1 1 -0.390625 0.234375 1 1 -0.390625 0.234375 1 1 -0.421875 0.234375 1 1 -0.421875 0.234375 1 1 -0.453125 0.234375 1 1 -0.453125 0.234375 1 1 -0.484375 0.234375 1 1 -0.484375 0.234375 1 1 -0.515625 0.234375 1 1 -0.515625 0.234375 1 1 -0.546875 0.234375 1 1 -0.546875 0.234375 1 1 -0.578125 0.234375 1 1 -0.578125 0.234375 1 1 -0.609375 0.234375 1 1 -0.609375 0.234375 1 1 -0.640625 0.234375 1 1 -0.640625 0.234375 1 1 -0.671875 0.234375 1 1 -0.671875 0.234375 1 1 -0.703125 0.234375 1 1 -0.703125 0.234375 1 1 -0.734375 0.234375 1 1 -0.734375 0.234375 1 1 -0.765625 0.234375 1 1 -0.765625 0.234375 1 1 -0.796875 0.234375 1 1 -0.796875 0.234375 1 1 -0.828125 0.234375 1 1 -0.828125 0.234375 1 1 -0.859375 0.234375 1 1 -0.859375 0.234375 1 1 -0.890625 0.234375 1 1 -0.890625 0.234375 1 1 -0.921875 0.234375 1 1 -0.921875 0.234375 1 1 -0.953125 0.234375 1 1 -0.953125 0.234375 1 1 -0.984375 0.234375 1 1 -0.984375 0.234375 1 1 -0.015625 0.265625 1 1 -0.015625 0.265625 1 1 -0.046875 0.265625 1 1 -0.046875 0.265625 1 1 -0.078125 0.265625 1 1 -0.078125 0.265625 1 1 -0.109375 0.265625 1 1 -0.109375 0.265625 1 1 -0.140625 0.265625 1 1 -0.140625 0.265625 1 1 -0.171875 0.265625 1 1 -0.171875 0.265625 1 1 -0.203125 0.265625 1 1 -0.203125 0.265625 1 1 -0.234375 0.265625 1 1 -0.234375 0.265625 1 1 -0.265625 0.265625 1 1 -0.265625 0.265625 1 1 -0.296875 0.265625 1 1 -0.296875 0.265625 1 1 -0.328125 0.265625 1 1 -0.328125 0.265625 1 1 -0.359375 0.265625 1 1 -0.359375 0.265625 1 1 -0.390625 0.265625 1 1 -0.390625 0.265625 1 1 -0.421875 0.265625 1 1 -0.421875 0.265625 1 1 -0.453125 0.265625 1 1 -0.453125 0.265625 1 1 -0.484375 0.265625 1 1 -0.484375 0.265625 1 1 -0.515625 0.265625 1 1 -0.515625 0.265625 1 1 -0.546875 0.265625 1 1 -0.546875 0.265625 1 1 -0.578125 0.265625 1 1 -0.578125 0.265625 1 1 -0.609375 0.265625 1 1 -0.609375 0.265625 1 1 -0.640625 0.265625 1 1 -0.640625 0.265625 1 1 -0.671875 0.265625 1 1 -0.671875 0.265625 1 1 -0.703125 0.265625 1 1 -0.703125 0.265625 1 1 -0.734375 0.265625 1 1 -0.734375 0.265625 1 1 -0.765625 0.265625 1 1 -0.765625 0.265625 1 1 -0.796875 0.265625 1 1 -0.796875 0.265625 1 1 -0.828125 0.265625 1 1 -0.828125 0.265625 1 1 -0.859375 0.265625 1 1 -0.859375 0.265625 1 1 -0.890625 0.265625 1 1 -0.890625 0.265625 1 1 -0.921875 0.265625 1 1 -0.921875 0.265625 1 1 -0.953125 0.265625 1 1 -0.953125 0.265625 1 1 -0.984375 0.265625 1 1 -0.984375 0.265625 1 1 -0.015625 0.296875 1 1 -0.015625 0.296875 1 1 -0.046875 0.296875 1 1 -0.046875 0.296875 1 1 -0.078125 0.296875 1 1 -0.078125 0.296875 1 1 -0.109375 0.296875 1 1 -0.109375 0.296875 1 1 -0.140625 0.296875 1 1 -0.140625 0.296875 1 1 -0.171875 0.296875 1 1 -0.171875 0.296875 1 1 -0.203125 0.296875 1 1 -0.203125 0.296875 1 1 -0.234375 0.296875 1 1 -0.234375 0.296875 1 1 -0.265625 0.296875 1 1 -0.265625 0.296875 1 1 -0.296875 0.296875 1 1 -0.296875 0.296875 1 1 -0.328125 0.296875 1 1 -0.328125 0.296875 1 1 -0.359375 0.296875 1 1 -0.359375 0.296875 1 1 -0.390625 0.296875 1 1 -0.390625 0.296875 1 1 -0.421875 0.296875 1 1 -0.421875 0.296875 1 1 -0.453125 0.296875 1 1 -0.453125 0.296875 1 1 -0.484375 0.296875 1 1 -0.484375 0.296875 1 1 -0.515625 0.296875 1 1 -0.515625 0.296875 1 1 -0.546875 0.296875 1 1 -0.546875 0.296875 1 1 -0.578125 0.296875 1 1 -0.578125 0.296875 1 1 -0.609375 0.296875 1 1 -0.609375 0.296875 1 1 -0.640625 0.296875 1 1 -0.640625 0.296875 1 1 -0.671875 0.296875 1 1 -0.671875 0.296875 1 1 -0.703125 0.296875 1 1 -0.703125 0.296875 1 1 -0.734375 0.296875 1 1 -0.734375 0.296875 1 1 -0.765625 0.296875 1 1 -0.765625 0.296875 1 1 -0.796875 0.296875 1 1 -0.796875 0.296875 1 1 -0.828125 0.296875 1 1 -0.828125 0.296875 1 1 -0.859375 0.296875 1 1 -0.859375 0.296875 1 1 -0.890625 0.296875 1 1 -0.890625 0.296875 1 1 -0.921875 0.296875 1 1 -0.921875 0.296875 1 1 -0.953125 0.296875 1 1 -0.953125 0.296875 1 1 -0.984375 0.296875 1 1 -0.984375 0.296875 1 1 -0.015625 0.328125 1 1 -0.015625 0.328125 1 1 -0.046875 0.328125 1 1 -0.046875 0.328125 1 1 -0.078125 0.328125 1 1 -0.078125 0.328125 1 1 -0.109375 0.328125 1 1 -0.109375 0.328125 1 1 -0.140625 0.328125 1 1 -0.140625 0.328125 1 1 -0.171875 0.328125 1 1 -0.171875 0.328125 1 1 -0.203125 0.328125 1 1 -0.203125 0.328125 1 1 -0.234375 0.328125 1 1 -0.234375 0.328125 1 1 -0.265625 0.328125 1 1 -0.265625 0.328125 1 1 -0.296875 0.328125 1 1 -0.296875 0.328125 1 1 -0.328125 0.328125 1 1 -0.328125 0.328125 1 1 -0.359375 0.328125 1 1 -0.359375 0.328125 1 1 -0.390625 0.328125 1 1 -0.390625 0.328125 1 1 -0.421875 0.328125 1 1 -0.421875 0.328125 1 1 -0.453125 0.328125 1 1 -0.453125 0.328125 1 1 -0.484375 0.328125 1 1 -0.484375 0.328125 1 1 -0.515625 0.328125 1 1 -0.515625 0.328125 1 1 -0.546875 0.328125 1 1 -0.546875 0.328125 1 1 -0.578125 0.328125 1 1 -0.578125 0.328125 1 1 -0.609375 0.328125 1 1 -0.609375 0.328125 1 1 -0.640625 0.328125 1 1 -0.640625 0.328125 1 1 -0.671875 0.328125 1 1 -0.671875 0.328125 1 1 -0.703125 0.328125 1 1 -0.703125 0.328125 1 1 -0.734375 0.328125 1 1 -0.734375 0.328125 1 1 -0.765625 0.328125 1 1 -0.765625 0.328125 1 1 -0.796875 0.328125 1 1 -0.796875 0.328125 1 1 -0.828125 0.328125 1 1 -0.828125 0.328125 1 1 -0.859375 0.328125 1 1 -0.859375 0.328125 1 1 -0.890625 0.328125 1 1 -0.890625 0.328125 1 1 -0.921875 0.328125 1 1 -0.921875 0.328125 1 1 -0.953125 0.328125 1 1 -0.953125 0.328125 1 1 -0.984375 0.328125 1 1 -0.984375 0.328125 1 1 -0.015625 0.359375 1 1 -0.015625 0.359375 1 1 -0.046875 0.359375 1 1 -0.046875 0.359375 1 1 -0.078125 0.359375 1 1 -0.078125 0.359375 1 1 -0.109375 0.359375 1 1 -0.109375 0.359375 1 1 -0.140625 0.359375 1 1 -0.140625 0.359375 1 1 -0.171875 0.359375 1 1 -0.171875 0.359375 1 1 -0.203125 0.359375 1 1 -0.203125 0.359375 1 1 -0.234375 0.359375 1 1 -0.234375 0.359375 1 1 -0.265625 0.359375 1 1 -0.265625 0.359375 1 1 -0.296875 0.359375 1 1 -0.296875 0.359375 1 1 -0.328125 0.359375 1 1 -0.328125 0.359375 1 1 -0.359375 0.359375 1 1 -0.359375 0.359375 1 1 -0.390625 0.359375 1 1 -0.390625 0.359375 1 1 -0.421875 0.359375 1 1 -0.421875 0.359375 1 1 -0.453125 0.359375 1 1 -0.453125 0.359375 1 1 -0.484375 0.359375 1 1 -0.484375 0.359375 1 1 -0.515625 0.359375 1 1 -0.515625 0.359375 1 1 -0.546875 0.359375 1 1 -0.546875 0.359375 1 1 -0.578125 0.359375 1 1 -0.578125 0.359375 1 1 -0.609375 0.359375 1 1 -0.609375 0.359375 1 1 -0.640625 0.359375 1 1 -0.640625 0.359375 1 1 -0.671875 0.359375 1 1 -0.671875 0.359375 1 1 -0.703125 0.359375 1 1 -0.703125 0.359375 1 1 -0.734375 0.359375 1 1 -0.734375 0.359375 1 1 -0.765625 0.359375 1 1 -0.765625 0.359375 1 1 -0.796875 0.359375 1 1 -0.796875 0.359375 1 1 -0.828125 0.359375 1 1 -0.828125 0.359375 1 1 -0.859375 0.359375 1 1 -0.859375 0.359375 1 1 -0.890625 0.359375 1 1 -0.890625 0.359375 1 1 -0.921875 0.359375 1 1 -0.921875 0.359375 1 1 -0.953125 0.359375 1 1 -0.953125 0.359375 1 1 -0.984375 0.359375 1 1 -0.984375 0.359375 1 1 -0.015625 0.390625 1 1 -0.015625 0.390625 1 1 -0.046875 0.390625 1 1 -0.046875 0.390625 1 1 -0.078125 0.390625 1 1 -0.078125 0.390625 1 1 -0.109375 0.390625 1 1 -0.109375 0.390625 1 1 -0.140625 0.390625 1 1 -0.140625 0.390625 1 1 -0.171875 0.390625 1 1 -0.171875 0.390625 1 1 -0.203125 0.390625 1 1 -0.203125 0.390625 1 1 -0.234375 0.390625 1 1 -0.234375 0.390625 1 1 -0.265625 0.390625 1 1 -0.265625 0.390625 1 1 -0.296875 0.390625 1 1 -0.296875 0.390625 1 1 -0.328125 0.390625 1 1 -0.328125 0.390625 1 1 -0.359375 0.390625 1 1 -0.359375 0.390625 1 1 -0.390625 0.390625 1 1 -0.390625 0.390625 1 1 -0.421875 0.390625 1 1 -0.421875 0.390625 1 1 -0.453125 0.390625 1 1 -0.453125 0.390625 1 1 -0.484375 0.390625 1 1 -0.484375 0.390625 1 1 -0.515625 0.390625 1 1 -0.515625 0.390625 1 1 -0.546875 0.390625 1 1 -0.546875 0.390625 1 1 -0.578125 0.390625 1 1 -0.578125 0.390625 1 1 -0.609375 0.390625 1 1 -0.609375 0.390625 1 1 -0.640625 0.390625 1 1 -0.640625 0.390625 1 1 -0.671875 0.390625 1 1 -0.671875 0.390625 1 1 -0.703125 0.390625 1 1 -0.703125 0.390625 1 1 -0.734375 0.390625 1 1 -0.734375 0.390625 1 1 -0.765625 0.390625 1 1 -0.765625 0.390625 1 1 -0.796875 0.390625 1 1 -0.796875 0.390625 1 1 -0.828125 0.390625 1 1 -0.828125 0.390625 1 1 -0.859375 0.390625 1 1 -0.859375 0.390625 1 1 -0.890625 0.390625 1 1 -0.890625 0.390625 1 1 -0.921875 0.390625 1 1 -0.921875 0.390625 1 1 -0.953125 0.390625 1 1 -0.953125 0.390625 1 1 -0.984375 0.390625 1 1 -0.984375 0.390625 1 1 -0.015625 0.421875 1 1 -0.015625 0.421875 1 1 -0.046875 0.421875 1 1 -0.046875 0.421875 1 1 -0.078125 0.421875 1 1 -0.078125 0.421875 1 1 -0.109375 0.421875 1 1 -0.109375 0.421875 1 1 -0.140625 0.421875 1 1 -0.140625 0.421875 1 1 -0.171875 0.421875 1 1 -0.171875 0.421875 1 1 -0.203125 0.421875 1 1 -0.203125 0.421875 1 1 -0.234375 0.421875 1 1 -0.234375 0.421875 1 1 -0.265625 0.421875 1 1 -0.265625 0.421875 1 1 -0.296875 0.421875 1 1 -0.296875 0.421875 1 1 -0.328125 0.421875 1 1 -0.328125 0.421875 1 1 -0.359375 0.421875 1 1 -0.359375 0.421875 1 1 -0.390625 0.421875 1 1 -0.390625 0.421875 1 1 -0.421875 0.421875 1 1 -0.421875 0.421875 1 1 -0.453125 0.421875 1 1 -0.453125 0.421875 1 1 -0.484375 0.421875 1 1 -0.484375 0.421875 1 1 -0.515625 0.421875 1 1 -0.515625 0.421875 1 1 -0.546875 0.421875 1 1 -0.546875 0.421875 1 1 -0.578125 0.421875 1 1 -0.578125 0.421875 1 1 -0.609375 0.421875 1 1 -0.609375 0.421875 1 1 -0.640625 0.421875 1 1 -0.640625 0.421875 1 1 -0.671875 0.421875 1 1 -0.671875 0.421875 1 1 -0.703125 0.421875 1 1 -0.703125 0.421875 1 1 -0.734375 0.421875 1 1 -0.734375 0.421875 1 1 -0.765625 0.421875 1 1 -0.765625 0.421875 1 1 -0.796875 0.421875 1 1 -0.796875 0.421875 1 1 -0.828125 0.421875 1 1 -0.828125 0.421875 1 1 -0.859375 0.421875 1 1 -0.859375 0.421875 1 1 -0.890625 0.421875 1 1 -0.890625 0.421875 1 1 -0.921875 0.421875 1 1 -0.921875 0.421875 1 1 -0.953125 0.421875 1 1 -0.953125 0.421875 1 1 -0.984375 0.421875 1 1 -0.984375 0.421875 1 1 -0.015625 0.453125 1 1 -0.015625 0.453125 1 1 -0.046875 0.453125 1 1 -0.046875 0.453125 1 1 -0.078125 0.453125 1 1 -0.078125 0.453125 1 1 -0.109375 0.453125 1 1 -0.109375 0.453125 1 1 -0.140625 0.453125 1 1 -0.140625 0.453125 1 1 -0.171875 0.453125 1 1 -0.171875 0.453125 1 1 -0.203125 0.453125 1 1 -0.203125 0.453125 1 1 -0.234375 0.453125 1 1 -0.234375 0.453125 1 1 -0.265625 0.453125 1 1 -0.265625 0.453125 1 1 -0.296875 0.453125 1 1 -0.296875 0.453125 1 1 -0.328125 0.453125 1 1 -0.328125 0.453125 1 1 -0.359375 0.453125 1 1 -0.359375 0.453125 1 1 -0.390625 0.453125 1 1 -0.390625 0.453125 1 1 -0.421875 0.453125 1 1 -0.421875 0.453125 1 1 -0.453125 0.453125 1 1 -0.453125 0.453125 1 1 -0.484375 0.453125 1 1 -0.484375 0.453125 1 1 -0.515625 0.453125 1 1 -0.515625 0.453125 1 1 -0.546875 0.453125 1 1 -0.546875 0.453125 1 1 -0.578125 0.453125 1 1 -0.578125 0.453125 1 1 -0.609375 0.453125 1 1 -0.609375 0.453125 1 1 -0.640625 0.453125 1 1 -0.640625 0.453125 1 1 -0.671875 0.453125 1 1 -0.671875 0.453125 1 1 -0.703125 0.453125 1 1 -0.703125 0.453125 1 1 -0.734375 0.453125 1 1 -0.734375 0.453125 1 1 -0.765625 0.453125 1 1 -0.765625 0.453125 1 1 -0.796875 0.453125 1 1 -0.796875 0.453125 1 1 -0.828125 0.453125 1 1 -0.828125 0.453125 1 1 -0.859375 0.453125 1 1 -0.859375 0.453125 1 1 -0.890625 0.453125 1 1 -0.890625 0.453125 1 1 -0.921875 0.453125 1 1 -0.921875 0.453125 1 1 -0.953125 0.453125 1 1 -0.953125 0.453125 1 1 -0.984375 0.453125 1 1 -0.984375 0.453125 1 1 -0.015625 0.484375 1 1 -0.015625 0.484375 1 1 -0.046875 0.484375 1 1 -0.046875 0.484375 1 1 -0.078125 0.484375 1 1 -0.078125 0.484375 1 1 -0.109375 0.484375 1 1 -0.109375 0.484375 1 1 -0.140625 0.484375 1 1 -0.140625 0.484375 1 1 -0.171875 0.484375 1 1 -0.171875 0.484375 1 1 -0.203125 0.484375 1 1 -0.203125 0.484375 1 1 -0.234375 0.484375 1 1 -0.234375 0.484375 1 1 -0.265625 0.484375 1 1 -0.265625 0.484375 1 1 -0.296875 0.484375 1 1 -0.296875 0.484375 1 1 -0.328125 0.484375 1 1 -0.328125 0.484375 1 1 -0.359375 0.484375 1 1 -0.359375 0.484375 1 1 -0.390625 0.484375 1 1 -0.390625 0.484375 1 1 -0.421875 0.484375 1 1 -0.421875 0.484375 1 1 -0.453125 0.484375 1 1 -0.453125 0.484375 1 1 -0.484375 0.484375 1 1 -0.484375 0.484375 1 1 -0.515625 0.484375 1 1 -0.515625 0.484375 1 1 -0.546875 0.484375 1 1 -0.546875 0.484375 1 1 -0.578125 0.484375 1 1 -0.578125 0.484375 1 1 -0.609375 0.484375 1 1 -0.609375 0.484375 1 1 -0.640625 0.484375 1 1 -0.640625 0.484375 1 1 -0.671875 0.484375 1 1 -0.671875 0.484375 1 1 -0.703125 0.484375 1 1 -0.703125 0.484375 1 1 -0.734375 0.484375 1 1 -0.734375 0.484375 1 1 -0.765625 0.484375 1 1 -0.765625 0.484375 1 1 -0.796875 0.484375 1 1 -0.796875 0.484375 1 1 -0.828125 0.484375 1 1 -0.828125 0.484375 1 1 -0.859375 0.484375 1 1 -0.859375 0.484375 1 1 -0.890625 0.484375 1 1 -0.890625 0.484375 1 1 -0.921875 0.484375 1 1 -0.921875 0.484375 1 1 -0.953125 0.484375 1 1 -0.953125 0.484375 1 1 -0.984375 0.484375 1 1 -0.984375 0.484375 1 1 -0.015625 0.515625 1 1 -0.015625 0.515625 1 1 -0.046875 0.515625 1 1 -0.046875 0.515625 1 1 -0.078125 0.515625 1 1 -0.078125 0.515625 1 1 -0.109375 0.515625 1 1 -0.109375 0.515625 1 1 -0.140625 0.515625 1 1 -0.140625 0.515625 1 1 -0.171875 0.515625 1 1 -0.171875 0.515625 1 1 -0.203125 0.515625 1 1 -0.203125 0.515625 1 1 -0.234375 0.515625 1 1 -0.234375 0.515625 1 1 -0.265625 0.515625 1 1 -0.265625 0.515625 1 1 -0.296875 0.515625 1 1 -0.296875 0.515625 1 1 -0.328125 0.515625 1 1 -0.328125 0.515625 1 1 -0.359375 0.515625 1 1 -0.359375 0.515625 1 1 -0.390625 0.515625 1 1 -0.390625 0.515625 1 1 -0.421875 0.515625 1 1 -0.421875 0.515625 1 1 -0.453125 0.515625 1 1 -0.453125 0.515625 1 1 -0.484375 0.515625 1 1 -0.484375 0.515625 1 1 -0.515625 0.515625 1 1 -0.515625 0.515625 1 1 -0.546875 0.515625 1 1 -0.546875 0.515625 1 1 -0.578125 0.515625 1 1 -0.578125 0.515625 1 1 -0.609375 0.515625 1 1 -0.609375 0.515625 1 1 -0.640625 0.515625 1 1 -0.640625 0.515625 1 1 -0.671875 0.515625 1 1 -0.671875 0.515625 1 1 -0.703125 0.515625 1 1 -0.703125 0.515625 1 1 -0.734375 0.515625 1 1 -0.734375 0.515625 1 1 -0.765625 0.515625 1 1 -0.765625 0.515625 1 1 -0.796875 0.515625 1 1 -0.796875 0.515625 1 1 -0.828125 0.515625 1 1 -0.828125 0.515625 1 1 -0.859375 0.515625 1 1 -0.859375 0.515625 1 1 -0.890625 0.515625 1 1 -0.890625 0.515625 1 1 -0.921875 0.515625 1 1 -0.921875 0.515625 1 1 -0.953125 0.515625 1 1 -0.953125 0.515625 1 1 -0.984375 0.515625 1 1 -0.984375 0.515625 1 1 -0.015625 0.546875 1 1 -0.015625 0.546875 1 1 -0.046875 0.546875 1 1 -0.046875 0.546875 1 1 -0.078125 0.546875 1 1 -0.078125 0.546875 1 1 -0.109375 0.546875 1 1 -0.109375 0.546875 1 1 -0.140625 0.546875 1 1 -0.140625 0.546875 1 1 -0.171875 0.546875 1 1 -0.171875 0.546875 1 1 -0.203125 0.546875 1 1 -0.203125 0.546875 1 1 -0.234375 0.546875 1 1 -0.234375 0.546875 1 1 -0.265625 0.546875 1 1 -0.265625 0.546875 1 1 -0.296875 0.546875 1 1 -0.296875 0.546875 1 1 -0.328125 0.546875 1 1 -0.328125 0.546875 1 1 -0.359375 0.546875 1 1 -0.359375 0.546875 1 1 -0.390625 0.546875 1 1 -0.390625 0.546875 1 1 -0.421875 0.546875 1 1 -0.421875 0.546875 1 1 -0.453125 0.546875 1 1 -0.453125 0.546875 1 1 -0.484375 0.546875 1 1 -0.484375 0.546875 1 1 -0.515625 0.546875 1 1 -0.515625 0.546875 1 1 -0.546875 0.546875 1 1 -0.546875 0.546875 1 1 -0.578125 0.546875 1 1 -0.578125 0.546875 1 1 -0.609375 0.546875 1 1 -0.609375 0.546875 1 1 -0.640625 0.546875 1 1 -0.640625 0.546875 1 1 -0.671875 0.546875 1 1 -0.671875 0.546875 1 1 -0.703125 0.546875 1 1 -0.703125 0.546875 1 1 -0.734375 0.546875 1 1 -0.734375 0.546875 1 1 -0.765625 0.546875 1 1 -0.765625 0.546875 1 1 -0.796875 0.546875 1 1 -0.796875 0.546875 1 1 -0.828125 0.546875 1 1 -0.828125 0.546875 1 1 -0.859375 0.546875 1 1 -0.859375 0.546875 1 1 -0.890625 0.546875 1 1 -0.890625 0.546875 1 1 -0.921875 0.546875 1 1 -0.921875 0.546875 1 1 -0.953125 0.546875 1 1 -0.953125 0.546875 1 1 -0.984375 0.546875 1 1 -0.984375 0.546875 1 1 -0.015625 0.578125 1 1 -0.015625 0.578125 1 1 -0.046875 0.578125 1 1 -0.046875 0.578125 1 1 -0.078125 0.578125 1 1 -0.078125 0.578125 1 1 -0.109375 0.578125 1 1 -0.109375 0.578125 1 1 -0.140625 0.578125 1 1 -0.140625 0.578125 1 1 -0.171875 0.578125 1 1 -0.171875 0.578125 1 1 -0.203125 0.578125 1 1 -0.203125 0.578125 1 1 -0.234375 0.578125 1 1 -0.234375 0.578125 1 1 -0.265625 0.578125 1 1 -0.265625 0.578125 1 1 -0.296875 0.578125 1 1 -0.296875 0.578125 1 1 -0.328125 0.578125 1 1 -0.328125 0.578125 1 1 -0.359375 0.578125 1 1 -0.359375 0.578125 1 1 -0.390625 0.578125 1 1 -0.390625 0.578125 1 1 -0.421875 0.578125 1 1 -0.421875 0.578125 1 1 -0.453125 0.578125 1 1 -0.453125 0.578125 1 1 -0.484375 0.578125 1 1 -0.484375 0.578125 1 1 -0.515625 0.578125 1 1 -0.515625 0.578125 1 1 -0.546875 0.578125 1 1 -0.546875 0.578125 1 1 -0.578125 0.578125 1 1 -0.578125 0.578125 1 1 -0.609375 0.578125 1 1 -0.609375 0.578125 1 1 -0.640625 0.578125 1 1 -0.640625 0.578125 1 1 -0.671875 0.578125 1 1 -0.671875 0.578125 1 1 -0.703125 0.578125 1 1 -0.703125 0.578125 1 1 -0.734375 0.578125 1 1 -0.734375 0.578125 1 1 -0.765625 0.578125 1 1 -0.765625 0.578125 1 1 -0.796875 0.578125 1 1 -0.796875 0.578125 1 1 -0.828125 0.578125 1 1 -0.828125 0.578125 1 1 -0.859375 0.578125 1 1 -0.859375 0.578125 1 1 -0.890625 0.578125 1 1 -0.890625 0.578125 1 1 -0.921875 0.578125 1 1 -0.921875 0.578125 1 1 -0.953125 0.578125 1 1 -0.953125 0.578125 1 1 -0.984375 0.578125 1 1 -0.984375 0.578125 1 1 -0.015625 0.609375 1 1 -0.015625 0.609375 1 1 -0.046875 0.609375 1 1 -0.046875 0.609375 1 1 -0.078125 0.609375 1 1 -0.078125 0.609375 1 1 -0.109375 0.609375 1 1 -0.109375 0.609375 1 1 -0.140625 0.609375 1 1 -0.140625 0.609375 1 1 -0.171875 0.609375 1 1 -0.171875 0.609375 1 1 -0.203125 0.609375 1 1 -0.203125 0.609375 1 1 -0.234375 0.609375 1 1 -0.234375 0.609375 1 1 -0.265625 0.609375 1 1 -0.265625 0.609375 1 1 -0.296875 0.609375 1 1 -0.296875 0.609375 1 1 -0.328125 0.609375 1 1 -0.328125 0.609375 1 1 -0.359375 0.609375 1 1 -0.359375 0.609375 1 1 -0.390625 0.609375 1 1 -0.390625 0.609375 1 1 -0.421875 0.609375 1 1 -0.421875 0.609375 1 1 -0.453125 0.609375 1 1 -0.453125 0.609375 1 1 -0.484375 0.609375 1 1 -0.484375 0.609375 1 1 -0.515625 0.609375 1 1 -0.515625 0.609375 1 1 -0.546875 0.609375 1 1 -0.546875 0.609375 1 1 -0.578125 0.609375 1 1 -0.578125 0.609375 1 1 -0.609375 0.609375 1 1 -0.609375 0.609375 1 1 -0.640625 0.609375 1 1 -0.640625 0.609375 1 1 -0.671875 0.609375 1 1 -0.671875 0.609375 1 1 -0.703125 0.609375 1 1 -0.703125 0.609375 1 1 -0.734375 0.609375 1 1 -0.734375 0.609375 1 1 -0.765625 0.609375 1 1 -0.765625 0.609375 1 1 -0.796875 0.609375 1 1 -0.796875 0.609375 1 1 -0.828125 0.609375 1 1 -0.828125 0.609375 1 1 -0.859375 0.609375 1 1 -0.859375 0.609375 1 1 -0.890625 0.609375 1 1 -0.890625 0.609375 1 1 -0.921875 0.609375 1 1 -0.921875 0.609375 1 1 -0.953125 0.609375 1 1 -0.953125 0.609375 1 1 -0.984375 0.609375 1 1 -0.984375 0.609375 1 1 -0.015625 0.640625 1 1 -0.015625 0.640625 1 1 -0.046875 0.640625 1 1 -0.046875 0.640625 1 1 -0.078125 0.640625 1 1 -0.078125 0.640625 1 1 -0.109375 0.640625 1 1 -0.109375 0.640625 1 1 -0.140625 0.640625 1 1 -0.140625 0.640625 1 1 -0.171875 0.640625 1 1 -0.171875 0.640625 1 1 -0.203125 0.640625 1 1 -0.203125 0.640625 1 1 -0.234375 0.640625 1 1 -0.234375 0.640625 1 1 -0.265625 0.640625 1 1 -0.265625 0.640625 1 1 -0.296875 0.640625 1 1 -0.296875 0.640625 1 1 -0.328125 0.640625 1 1 -0.328125 0.640625 1 1 -0.359375 0.640625 1 1 -0.359375 0.640625 1 1 -0.390625 0.640625 1 1 -0.390625 0.640625 1 1 -0.421875 0.640625 1 1 -0.421875 0.640625 1 1 -0.453125 0.640625 1 1 -0.453125 0.640625 1 1 -0.484375 0.640625 1 1 -0.484375 0.640625 1 1 -0.515625 0.640625 1 1 -0.515625 0.640625 1 1 -0.546875 0.640625 1 1 -0.546875 0.640625 1 1 -0.578125 0.640625 1 1 -0.578125 0.640625 1 1 -0.609375 0.640625 1 1 -0.609375 0.640625 1 1 -0.640625 0.640625 1 1 -0.640625 0.640625 1 1 -0.671875 0.640625 1 1 -0.671875 0.640625 1 1 -0.703125 0.640625 1 1 -0.703125 0.640625 1 1 -0.734375 0.640625 1 1 -0.734375 0.640625 1 1 -0.765625 0.640625 1 1 -0.765625 0.640625 1 1 -0.796875 0.640625 1 1 -0.796875 0.640625 1 1 -0.828125 0.640625 1 1 -0.828125 0.640625 1 1 -0.859375 0.640625 1 1 -0.859375 0.640625 1 1 -0.890625 0.640625 1 1 -0.890625 0.640625 1 1 -0.921875 0.640625 1 1 -0.921875 0.640625 1 1 -0.953125 0.640625 1 1 -0.953125 0.640625 1 1 -0.984375 0.640625 1 1 -0.984375 0.640625 1 1 -0.015625 0.671875 1 1 -0.015625 0.671875 1 1 -0.046875 0.671875 1 1 -0.046875 0.671875 1 1 -0.078125 0.671875 1 1 -0.078125 0.671875 1 1 -0.109375 0.671875 1 1 -0.109375 0.671875 1 1 -0.140625 0.671875 1 1 -0.140625 0.671875 1 1 -0.171875 0.671875 1 1 -0.171875 0.671875 1 1 -0.203125 0.671875 1 1 -0.203125 0.671875 1 1 -0.234375 0.671875 1 1 -0.234375 0.671875 1 1 -0.265625 0.671875 1 1 -0.265625 0.671875 1 1 -0.296875 0.671875 1 1 -0.296875 0.671875 1 1 -0.328125 0.671875 1 1 -0.328125 0.671875 1 1 -0.359375 0.671875 1 1 -0.359375 0.671875 1 1 -0.390625 0.671875 1 1 -0.390625 0.671875 1 1 -0.421875 0.671875 1 1 -0.421875 0.671875 1 1 -0.453125 0.671875 1 1 -0.453125 0.671875 1 1 -0.484375 0.671875 1 1 -0.484375 0.671875 1 1 -0.515625 0.671875 1 1 -0.515625 0.671875 1 1 -0.546875 0.671875 1 1 -0.546875 0.671875 1 1 -0.578125 0.671875 1 1 -0.578125 0.671875 1 1 -0.609375 0.671875 1 1 -0.609375 0.671875 1 1 -0.640625 0.671875 1 1 -0.640625 0.671875 1 1 -0.671875 0.671875 1 1 -0.671875 0.671875 1 1 -0.703125 0.671875 1 1 -0.703125 0.671875 1 1 -0.734375 0.671875 1 1 -0.734375 0.671875 1 1 -0.765625 0.671875 1 1 -0.765625 0.671875 1 1 -0.796875 0.671875 1 1 -0.796875 0.671875 1 1 -0.828125 0.671875 1 1 -0.828125 0.671875 1 1 -0.859375 0.671875 1 1 -0.859375 0.671875 1 1 -0.890625 0.671875 1 1 -0.890625 0.671875 1 1 -0.921875 0.671875 1 1 -0.921875 0.671875 1 1 -0.953125 0.671875 1 1 -0.953125 0.671875 1 1 -0.984375 0.671875 1 1 -0.984375 0.671875 1 1 -0.015625 0.703125 1 1 -0.015625 0.703125 1 1 -0.046875 0.703125 1 1 -0.046875 0.703125 1 1 -0.078125 0.703125 1 1 -0.078125 0.703125 1 1 -0.109375 0.703125 1 1 -0.109375 0.703125 1 1 -0.140625 0.703125 1 1 -0.140625 0.703125 1 1 -0.171875 0.703125 1 1 -0.171875 0.703125 1 1 -0.203125 0.703125 1 1 -0.203125 0.703125 1 1 -0.234375 0.703125 1 1 -0.234375 0.703125 1 1 -0.265625 0.703125 1 1 -0.265625 0.703125 1 1 -0.296875 0.703125 1 1 -0.296875 0.703125 1 1 -0.328125 0.703125 1 1 -0.328125 0.703125 1 1 -0.359375 0.703125 1 1 -0.359375 0.703125 1 1 -0.390625 0.703125 1 1 -0.390625 0.703125 1 1 -0.421875 0.703125 1 1 -0.421875 0.703125 1 1 -0.453125 0.703125 1 1 -0.453125 0.703125 1 1 -0.484375 0.703125 1 1 -0.484375 0.703125 1 1 -0.515625 0.703125 1 1 -0.515625 0.703125 1 1 -0.546875 0.703125 1 1 -0.546875 0.703125 1 1 -0.578125 0.703125 1 1 -0.578125 0.703125 1 1 -0.609375 0.703125 1 1 -0.609375 0.703125 1 1 -0.640625 0.703125 1 1 -0.640625 0.703125 1 1 -0.671875 0.703125 1 1 -0.671875 0.703125 1 1 -0.703125 0.703125 1 1 -0.703125 0.703125 1 1 -0.734375 0.703125 1 1 -0.734375 0.703125 1 1 -0.765625 0.703125 1 1 -0.765625 0.703125 1 1 -0.796875 0.703125 1 1 -0.796875 0.703125 1 1 -0.828125 0.703125 1 1 -0.828125 0.703125 1 1 -0.859375 0.703125 1 1 -0.859375 0.703125 1 1 -0.890625 0.703125 1 1 -0.890625 0.703125 1 1 -0.921875 0.703125 1 1 -0.921875 0.703125 1 1 -0.953125 0.703125 1 1 -0.953125 0.703125 1 1 -0.984375 0.703125 1 1 -0.984375 0.703125 1 1 -0.015625 0.734375 1 1 -0.015625 0.734375 1 1 -0.046875 0.734375 1 1 -0.046875 0.734375 1 1 -0.078125 0.734375 1 1 -0.078125 0.734375 1 1 -0.109375 0.734375 1 1 -0.109375 0.734375 1 1 -0.140625 0.734375 1 1 -0.140625 0.734375 1 1 -0.171875 0.734375 1 1 -0.171875 0.734375 1 1 -0.203125 0.734375 1 1 -0.203125 0.734375 1 1 -0.234375 0.734375 1 1 -0.234375 0.734375 1 1 -0.265625 0.734375 1 1 -0.265625 0.734375 1 1 -0.296875 0.734375 1 1 -0.296875 0.734375 1 1 -0.328125 0.734375 1 1 -0.328125 0.734375 1 1 -0.359375 0.734375 1 1 -0.359375 0.734375 1 1 -0.390625 0.734375 1 1 -0.390625 0.734375 1 1 -0.421875 0.734375 1 1 -0.421875 0.734375 1 1 -0.453125 0.734375 1 1 -0.453125 0.734375 1 1 -0.484375 0.734375 1 1 -0.484375 0.734375 1 1 -0.515625 0.734375 1 1 -0.515625 0.734375 1 1 -0.546875 0.734375 1 1 -0.546875 0.734375 1 1 -0.578125 0.734375 1 1 -0.578125 0.734375 1 1 -0.609375 0.734375 1 1 -0.609375 0.734375 1 1 -0.640625 0.734375 1 1 -0.640625 0.734375 1 1 -0.671875 0.734375 1 1 -0.671875 0.734375 1 1 -0.703125 0.734375 1 1 -0.703125 0.734375 1 1 -0.734375 0.734375 1 1 -0.734375 0.734375 1 1 -0.765625 0.734375 1 1 -0.765625 0.734375 1 1 -0.796875 0.734375 1 1 -0.796875 0.734375 1 1 -0.828125 0.734375 1 1 -0.828125 0.734375 1 1 -0.859375 0.734375 1 1 -0.859375 0.734375 1 1 -0.890625 0.734375 1 1 -0.890625 0.734375 1 1 -0.921875 0.734375 1 1 -0.921875 0.734375 1 1 -0.953125 0.734375 1 1 -0.953125 0.734375 1 1 -0.984375 0.734375 1 1 -0.984375 0.734375 1 1 -0.015625 0.765625 1 1 -0.015625 0.765625 1 1 -0.046875 0.765625 1 1 -0.046875 0.765625 1 1 -0.078125 0.765625 1 1 -0.078125 0.765625 1 1 -0.109375 0.765625 1 1 -0.109375 0.765625 1 1 -0.140625 0.765625 1 1 -0.140625 0.765625 1 1 -0.171875 0.765625 1 1 -0.171875 0.765625 1 1 -0.203125 0.765625 1 1 -0.203125 0.765625 1 1 -0.234375 0.765625 1 1 -0.234375 0.765625 1 1 -0.265625 0.765625 1 1 -0.265625 0.765625 1 1 -0.296875 0.765625 1 1 -0.296875 0.765625 1 1 -0.328125 0.765625 1 1 -0.328125 0.765625 1 1 -0.359375 0.765625 1 1 -0.359375 0.765625 1 1 -0.390625 0.765625 1 1 -0.390625 0.765625 1 1 -0.421875 0.765625 1 1 -0.421875 0.765625 1 1 -0.453125 0.765625 1 1 -0.453125 0.765625 1 1 -0.484375 0.765625 1 1 -0.484375 0.765625 1 1 -0.515625 0.765625 1 1 -0.515625 0.765625 1 1 -0.546875 0.765625 1 1 -0.546875 0.765625 1 1 -0.578125 0.765625 1 1 -0.578125 0.765625 1 1 -0.609375 0.765625 1 1 -0.609375 0.765625 1 1 -0.640625 0.765625 1 1 -0.640625 0.765625 1 1 -0.671875 0.765625 1 1 -0.671875 0.765625 1 1 -0.703125 0.765625 1 1 -0.703125 0.765625 1 1 -0.734375 0.765625 1 1 -0.734375 0.765625 1 1 -0.765625 0.765625 1 1 -0.765625 0.765625 1 1 -0.796875 0.765625 1 1 -0.796875 0.765625 1 1 -0.828125 0.765625 1 1 -0.828125 0.765625 1 1 -0.859375 0.765625 1 1 -0.859375 0.765625 1 1 -0.890625 0.765625 1 1 -0.890625 0.765625 1 1 -0.921875 0.765625 1 1 -0.921875 0.765625 1 1 -0.953125 0.765625 1 1 -0.953125 0.765625 1 1 -0.984375 0.765625 1 1 -0.984375 0.765625 1 1 -0.015625 0.796875 1 1 -0.015625 0.796875 1 1 -0.046875 0.796875 1 1 -0.046875 0.796875 1 1 -0.078125 0.796875 1 1 -0.078125 0.796875 1 1 -0.109375 0.796875 1 1 -0.109375 0.796875 1 1 -0.140625 0.796875 1 1 -0.140625 0.796875 1 1 -0.171875 0.796875 1 1 -0.171875 0.796875 1 1 -0.203125 0.796875 1 1 -0.203125 0.796875 1 1 -0.234375 0.796875 1 1 -0.234375 0.796875 1 1 -0.265625 0.796875 1 1 -0.265625 0.796875 1 1 -0.296875 0.796875 1 1 -0.296875 0.796875 1 1 -0.328125 0.796875 1 1 -0.328125 0.796875 1 1 -0.359375 0.796875 1 1 -0.359375 0.796875 1 1 -0.390625 0.796875 1 1 -0.390625 0.796875 1 1 -0.421875 0.796875 1 1 -0.421875 0.796875 1 1 -0.453125 0.796875 1 1 -0.453125 0.796875 1 1 -0.484375 0.796875 1 1 -0.484375 0.796875 1 1 -0.515625 0.796875 1 1 -0.515625 0.796875 1 1 -0.546875 0.796875 1 1 -0.546875 0.796875 1 1 -0.578125 0.796875 1 1 -0.578125 0.796875 1 1 -0.609375 0.796875 1 1 -0.609375 0.796875 1 1 -0.640625 0.796875 1 1 -0.640625 0.796875 1 1 -0.671875 0.796875 1 1 -0.671875 0.796875 1 1 -0.703125 0.796875 1 1 -0.703125 0.796875 1 1 -0.734375 0.796875 1 1 -0.734375 0.796875 1 1 -0.765625 0.796875 1 1 -0.765625 0.796875 1 1 -0.796875 0.796875 1 1 -0.796875 0.796875 1 1 -0.828125 0.796875 1 1 -0.828125 0.796875 1 1 -0.859375 0.796875 1 1 -0.859375 0.796875 1 1 -0.890625 0.796875 1 1 -0.890625 0.796875 1 1 -0.921875 0.796875 1 1 -0.921875 0.796875 1 1 -0.953125 0.796875 1 1 -0.953125 0.796875 1 1 -0.984375 0.796875 1 1 -0.984375 0.796875 1 1 -0.015625 0.828125 1 1 -0.015625 0.828125 1 1 -0.046875 0.828125 1 1 -0.046875 0.828125 1 1 -0.078125 0.828125 1 1 -0.078125 0.828125 1 1 -0.109375 0.828125 1 1 -0.109375 0.828125 1 1 -0.140625 0.828125 1 1 -0.140625 0.828125 1 1 -0.171875 0.828125 1 1 -0.171875 0.828125 1 1 -0.203125 0.828125 1 1 -0.203125 0.828125 1 1 -0.234375 0.828125 1 1 -0.234375 0.828125 1 1 -0.265625 0.828125 1 1 -0.265625 0.828125 1 1 -0.296875 0.828125 1 1 -0.296875 0.828125 1 1 -0.328125 0.828125 1 1 -0.328125 0.828125 1 1 -0.359375 0.828125 1 1 -0.359375 0.828125 1 1 -0.390625 0.828125 1 1 -0.390625 0.828125 1 1 -0.421875 0.828125 1 1 -0.421875 0.828125 1 1 -0.453125 0.828125 1 1 -0.453125 0.828125 1 1 -0.484375 0.828125 1 1 -0.484375 0.828125 1 1 -0.515625 0.828125 1 1 -0.515625 0.828125 1 1 -0.546875 0.828125 1 1 -0.546875 0.828125 1 1 -0.578125 0.828125 1 1 -0.578125 0.828125 1 1 -0.609375 0.828125 1 1 -0.609375 0.828125 1 1 -0.640625 0.828125 1 1 -0.640625 0.828125 1 1 -0.671875 0.828125 1 1 -0.671875 0.828125 1 1 -0.703125 0.828125 1 1 -0.703125 0.828125 1 1 -0.734375 0.828125 1 1 -0.734375 0.828125 1 1 -0.765625 0.828125 1 1 -0.765625 0.828125 1 1 -0.796875 0.828125 1 1 -0.796875 0.828125 1 1 -0.828125 0.828125 1 1 -0.828125 0.828125 1 1 -0.859375 0.828125 1 1 -0.859375 0.828125 1 1 -0.890625 0.828125 1 1 -0.890625 0.828125 1 1 -0.921875 0.828125 1 1 -0.921875 0.828125 1 1 -0.953125 0.828125 1 1 -0.953125 0.828125 1 1 -0.984375 0.828125 1 1 -0.984375 0.828125 1 1 -0.015625 0.859375 1 1 -0.015625 0.859375 1 1 -0.046875 0.859375 1 1 -0.046875 0.859375 1 1 -0.078125 0.859375 1 1 -0.078125 0.859375 1 1 -0.109375 0.859375 1 1 -0.109375 0.859375 1 1 -0.140625 0.859375 1 1 -0.140625 0.859375 1 1 -0.171875 0.859375 1 1 -0.171875 0.859375 1 1 -0.203125 0.859375 1 1 -0.203125 0.859375 1 1 -0.234375 0.859375 1 1 -0.234375 0.859375 1 1 -0.265625 0.859375 1 1 -0.265625 0.859375 1 1 -0.296875 0.859375 1 1 -0.296875 0.859375 1 1 -0.328125 0.859375 1 1 -0.328125 0.859375 1 1 -0.359375 0.859375 1 1 -0.359375 0.859375 1 1 -0.390625 0.859375 1 1 -0.390625 0.859375 1 1 -0.421875 0.859375 1 1 -0.421875 0.859375 1 1 -0.453125 0.859375 1 1 -0.453125 0.859375 1 1 -0.484375 0.859375 1 1 -0.484375 0.859375 1 1 -0.515625 0.859375 1 1 -0.515625 0.859375 1 1 -0.546875 0.859375 1 1 -0.546875 0.859375 1 1 -0.578125 0.859375 1 1 -0.578125 0.859375 1 1 -0.609375 0.859375 1 1 -0.609375 0.859375 1 1 -0.640625 0.859375 1 1 -0.640625 0.859375 1 1 -0.671875 0.859375 1 1 -0.671875 0.859375 1 1 -0.703125 0.859375 1 1 -0.703125 0.859375 1 1 -0.734375 0.859375 1 1 -0.734375 0.859375 1 1 -0.765625 0.859375 1 1 -0.765625 0.859375 1 1 -0.796875 0.859375 1 1 -0.796875 0.859375 1 1 -0.828125 0.859375 1 1 -0.828125 0.859375 1 1 -0.859375 0.859375 1 1 -0.859375 0.859375 1 1 -0.890625 0.859375 1 1 -0.890625 0.859375 1 1 -0.921875 0.859375 1 1 -0.921875 0.859375 1 1 -0.953125 0.859375 1 1 -0.953125 0.859375 1 1 -0.984375 0.859375 1 1 -0.984375 0.859375 1 1 -0.015625 0.890625 1 1 -0.015625 0.890625 1 1 -0.046875 0.890625 1 1 -0.046875 0.890625 1 1 -0.078125 0.890625 1 1 -0.078125 0.890625 1 1 -0.109375 0.890625 1 1 -0.109375 0.890625 1 1 -0.140625 0.890625 1 1 -0.140625 0.890625 1 1 -0.171875 0.890625 1 1 -0.171875 0.890625 1 1 -0.203125 0.890625 1 1 -0.203125 0.890625 1 1 -0.234375 0.890625 1 1 -0.234375 0.890625 1 1 -0.265625 0.890625 1 1 -0.265625 0.890625 1 1 -0.296875 0.890625 1 1 -0.296875 0.890625 1 1 -0.328125 0.890625 1 1 -0.328125 0.890625 1 1 -0.359375 0.890625 1 1 -0.359375 0.890625 1 1 -0.390625 0.890625 1 1 -0.390625 0.890625 1 1 -0.421875 0.890625 1 1 -0.421875 0.890625 1 1 -0.453125 0.890625 1 1 -0.453125 0.890625 1 1 -0.484375 0.890625 1 1 -0.484375 0.890625 1 1 -0.515625 0.890625 1 1 -0.515625 0.890625 1 1 -0.546875 0.890625 1 1 -0.546875 0.890625 1 1 -0.578125 0.890625 1 1 -0.578125 0.890625 1 1 -0.609375 0.890625 1 1 -0.609375 0.890625 1 1 -0.640625 0.890625 1 1 -0.640625 0.890625 1 1 -0.671875 0.890625 1 1 -0.671875 0.890625 1 1 -0.703125 0.890625 1 1 -0.703125 0.890625 1 1 -0.734375 0.890625 1 1 -0.734375 0.890625 1 1 -0.765625 0.890625 1 1 -0.765625 0.890625 1 1 -0.796875 0.890625 1 1 -0.796875 0.890625 1 1 -0.828125 0.890625 1 1 -0.828125 0.890625 1 1 -0.859375 0.890625 1 1 -0.859375 0.890625 1 1 -0.890625 0.890625 1 1 -0.890625 0.890625 1 1 -0.921875 0.890625 1 1 -0.921875 0.890625 1 1 -0.953125 0.890625 1 1 -0.953125 0.890625 1 1 -0.984375 0.890625 1 1 -0.984375 0.890625 1 1 -0.015625 0.921875 1 1 -0.015625 0.921875 1 1 -0.046875 0.921875 1 1 -0.046875 0.921875 1 1 -0.078125 0.921875 1 1 -0.078125 0.921875 1 1 -0.109375 0.921875 1 1 -0.109375 0.921875 1 1 -0.140625 0.921875 1 1 -0.140625 0.921875 1 1 -0.171875 0.921875 1 1 -0.171875 0.921875 1 1 -0.203125 0.921875 1 1 -0.203125 0.921875 1 1 -0.234375 0.921875 1 1 -0.234375 0.921875 1 1 -0.265625 0.921875 1 1 -0.265625 0.921875 1 1 -0.296875 0.921875 1 1 -0.296875 0.921875 1 1 -0.328125 0.921875 1 1 -0.328125 0.921875 1 1 -0.359375 0.921875 1 1 -0.359375 0.921875 1 1 -0.390625 0.921875 1 1 -0.390625 0.921875 1 1 -0.421875 0.921875 1 1 -0.421875 0.921875 1 1 -0.453125 0.921875 1 1 -0.453125 0.921875 1 1 -0.484375 0.921875 1 1 -0.484375 0.921875 1 1 -0.515625 0.921875 1 1 -0.515625 0.921875 1 1 -0.546875 0.921875 1 1 -0.546875 0.921875 1 1 -0.578125 0.921875 1 1 -0.578125 0.921875 1 1 -0.609375 0.921875 1 1 -0.609375 0.921875 1 1 -0.640625 0.921875 1 1 -0.640625 0.921875 1 1 -0.671875 0.921875 1 1 -0.671875 0.921875 1 1 -0.703125 0.921875 1 1 -0.703125 0.921875 1 1 -0.734375 0.921875 1 1 -0.734375 0.921875 1 1 -0.765625 0.921875 1 1 -0.765625 0.921875 1 1 -0.796875 0.921875 1 1 -0.796875 0.921875 1 1 -0.828125 0.921875 1 1 -0.828125 0.921875 1 1 -0.859375 0.921875 1 1 -0.859375 0.921875 1 1 -0.890625 0.921875 1 1 -0.890625 0.921875 1 1 -0.921875 0.921875 1 1 -0.921875 0.921875 1 1 -0.953125 0.921875 1 1 -0.953125 0.921875 1 1 -0.984375 0.921875 1 1 -0.984375 0.921875 1 1 -0.015625 0.953125 1 1 -0.015625 0.953125 1 1 -0.046875 0.953125 1 1 -0.046875 0.953125 1 1 -0.078125 0.953125 1 1 -0.078125 0.953125 1 1 -0.109375 0.953125 1 1 -0.109375 0.953125 1 1 -0.140625 0.953125 1 1 -0.140625 0.953125 1 1 -0.171875 0.953125 1 1 -0.171875 0.953125 1 1 -0.203125 0.953125 1 1 -0.203125 0.953125 1 1 -0.234375 0.953125 1 1 -0.234375 0.953125 1 1 -0.265625 0.953125 1 1 -0.265625 0.953125 1 1 -0.296875 0.953125 1 1 -0.296875 0.953125 1 1 -0.328125 0.953125 1 1 -0.328125 0.953125 1 1 -0.359375 0.953125 1 1 -0.359375 0.953125 1 1 -0.390625 0.953125 1 1 -0.390625 0.953125 1 1 -0.421875 0.953125 1 1 -0.421875 0.953125 1 1 -0.453125 0.953125 1 1 -0.453125 0.953125 1 1 -0.484375 0.953125 1 1 -0.484375 0.953125 1 1 -0.515625 0.953125 1 1 -0.515625 0.953125 1 1 -0.546875 0.953125 1 1 -0.546875 0.953125 1 1 -0.578125 0.953125 1 1 -0.578125 0.953125 1 1 -0.609375 0.953125 1 1 -0.609375 0.953125 1 1 -0.640625 0.953125 1 1 -0.640625 0.953125 1 1 -0.671875 0.953125 1 1 -0.671875 0.953125 1 1 -0.703125 0.953125 1 1 -0.703125 0.953125 1 1 -0.734375 0.953125 1 1 -0.734375 0.953125 1 1 -0.765625 0.953125 1 1 -0.765625 0.953125 1 1 -0.796875 0.953125 1 1 -0.796875 0.953125 1 1 -0.828125 0.953125 1 1 -0.828125 0.953125 1 1 -0.859375 0.953125 1 1 -0.859375 0.953125 1 1 -0.890625 0.953125 1 1 -0.890625 0.953125 1 1 -0.921875 0.953125 1 1 -0.921875 0.953125 1 1 -0.953125 0.953125 1 1 -0.953125 0.953125 1 1 -0.984375 0.953125 1 1 -0.984375 0.953125 1 1 -0.015625 0.984375 1 1 -0.015625 0.984375 1 1 -0.046875 0.984375 1 1 -0.046875 0.984375 1 1 -0.078125 0.984375 1 1 -0.078125 0.984375 1 1 -0.109375 0.984375 1 1 -0.109375 0.984375 1 1 -0.140625 0.984375 1 1 -0.140625 0.984375 1 1 -0.171875 0.984375 1 1 -0.171875 0.984375 1 1 -0.203125 0.984375 1 1 -0.203125 0.984375 1 1 -0.234375 0.984375 1 1 -0.234375 0.984375 1 1 -0.265625 0.984375 1 1 -0.265625 0.984375 1 1 -0.296875 0.984375 1 1 -0.296875 0.984375 1 1 -0.328125 0.984375 1 1 -0.328125 0.984375 1 1 -0.359375 0.984375 1 1 -0.359375 0.984375 1 1 -0.390625 0.984375 1 1 -0.390625 0.984375 1 1 -0.421875 0.984375 1 1 -0.421875 0.984375 1 1 -0.453125 0.984375 1 1 -0.453125 0.984375 1 1 -0.484375 0.984375 1 1 -0.484375 0.984375 1 1 -0.515625 0.984375 1 1 -0.515625 0.984375 1 1 -0.546875 0.984375 1 1 -0.546875 0.984375 1 1 -0.578125 0.984375 1 1 -0.578125 0.984375 1 1 -0.609375 0.984375 1 1 -0.609375 0.984375 1 1 -0.640625 0.984375 1 1 -0.640625 0.984375 1 1 -0.671875 0.984375 1 1 -0.671875 0.984375 1 1 -0.703125 0.984375 1 1 -0.703125 0.984375 1 1 -0.734375 0.984375 1 1 -0.734375 0.984375 1 1 -0.765625 0.984375 1 1 -0.765625 0.984375 1 1 -0.796875 0.984375 1 1 -0.796875 0.984375 1 1 -0.828125 0.984375 1 1 -0.828125 0.984375 1 1 -0.859375 0.984375 1 1 -0.859375 0.984375 1 1 -0.890625 0.984375 1 1 -0.890625 0.984375 1 1 -0.921875 0.984375 1 1 -0.921875 0.984375 1 1 -0.953125 0.984375 1 1 -0.953125 0.984375 1 1 -0.984375 0.984375 1 1 -0.984375 0.984375 1 1 -0.03125 0.03125 1 1 -0.03125 0.03125 1 1 -0.09375 0.03125 1 1 -0.09375 0.03125 1 1 -0.15625 0.03125 1 1 -0.15625 0.03125 1 1 -0.21875 0.03125 1 1 -0.21875 0.03125 1 1 -0.28125 0.03125 1 1 -0.28125 0.03125 1 1 -0.34375 0.03125 1 1 -0.34375 0.03125 1 1 -0.40625 0.03125 1 1 -0.40625 0.03125 1 1 -0.46875 0.03125 1 1 -0.46875 0.03125 1 1 -0.53125 0.03125 1 1 -0.53125 0.03125 1 1 -0.59375 0.03125 1 1 -0.59375 0.03125 1 1 -0.65625 0.03125 1 1 -0.65625 0.03125 1 1 -0.71875 0.03125 1 1 -0.71875 0.03125 1 1 -0.78125 0.03125 1 1 -0.78125 0.03125 1 1 -0.84375 0.03125 1 1 -0.84375 0.03125 1 1 -0.90625 0.03125 1 1 -0.90625 0.03125 1 1 -0.96875 0.03125 1 1 -0.96875 0.03125 1 1 -0.03125 0.09375 1 1 -0.03125 0.09375 1 1 -0.09375 0.09375 1 1 -0.09375 0.09375 1 1 -0.15625 0.09375 1 1 -0.15625 0.09375 1 1 -0.21875 0.09375 1 1 -0.21875 0.09375 1 1 -0.28125 0.09375 1 1 -0.28125 0.09375 1 1 -0.34375 0.09375 1 1 -0.34375 0.09375 1 1 -0.40625 0.09375 1 1 -0.40625 0.09375 1 1 -0.46875 0.09375 1 1 -0.46875 0.09375 1 1 -0.53125 0.09375 1 1 -0.53125 0.09375 1 1 -0.59375 0.09375 1 1 -0.59375 0.09375 1 1 -0.65625 0.09375 1 1 -0.65625 0.09375 1 1 -0.71875 0.09375 1 1 -0.71875 0.09375 1 1 -0.78125 0.09375 1 1 -0.78125 0.09375 1 1 -0.84375 0.09375 1 1 -0.84375 0.09375 1 1 -0.90625 0.09375 1 1 -0.90625 0.09375 1 1 -0.96875 0.09375 1 1 -0.96875 0.09375 1 1 -0.03125 0.15625 1 1 -0.03125 0.15625 1 1 -0.09375 0.15625 1 1 -0.09375 0.15625 1 1 -0.15625 0.15625 1 1 -0.15625 0.15625 1 1 -0.21875 0.15625 1 1 -0.21875 0.15625 1 1 -0.28125 0.15625 1 1 -0.28125 0.15625 1 1 -0.34375 0.15625 1 1 -0.34375 0.15625 1 1 -0.40625 0.15625 1 1 -0.40625 0.15625 1 1 -0.46875 0.15625 1 1 -0.46875 0.15625 1 1 -0.53125 0.15625 1 1 -0.53125 0.15625 1 1 -0.59375 0.15625 1 1 -0.59375 0.15625 1 1 -0.65625 0.15625 1 1 -0.65625 0.15625 1 1 -0.71875 0.15625 1 1 -0.71875 0.15625 1 1 -0.78125 0.15625 1 1 -0.78125 0.15625 1 1 -0.84375 0.15625 1 1 -0.84375 0.15625 1 1 -0.90625 0.15625 1 1 -0.90625 0.15625 1 1 -0.96875 0.15625 1 1 -0.96875 0.15625 1 1 -0.03125 0.21875 1 1 -0.03125 0.21875 1 1 -0.09375 0.21875 1 1 -0.09375 0.21875 1 1 -0.15625 0.21875 1 1 -0.15625 0.21875 1 1 -0.21875 0.21875 1 1 -0.21875 0.21875 1 1 -0.28125 0.21875 1 1 -0.28125 0.21875 1 1 -0.34375 0.21875 1 1 -0.34375 0.21875 1 1 -0.40625 0.21875 1 1 -0.40625 0.21875 1 1 -0.46875 0.21875 1 1 -0.46875 0.21875 1 1 -0.53125 0.21875 1 1 -0.53125 0.21875 1 1 -0.59375 0.21875 1 1 -0.59375 0.21875 1 1 -0.65625 0.21875 1 1 -0.65625 0.21875 1 1 -0.71875 0.21875 1 1 -0.71875 0.21875 1 1 -0.78125 0.21875 1 1 -0.78125 0.21875 1 1 -0.84375 0.21875 1 1 -0.84375 0.21875 1 1 -0.90625 0.21875 1 1 -0.90625 0.21875 1 1 -0.96875 0.21875 1 1 -0.96875 0.21875 1 1 -0.03125 0.28125 1 1 -0.03125 0.28125 1 1 -0.09375 0.28125 1 1 -0.09375 0.28125 1 1 -0.15625 0.28125 1 1 -0.15625 0.28125 1 1 -0.21875 0.28125 1 1 -0.21875 0.28125 1 1 -0.28125 0.28125 1 1 -0.28125 0.28125 1 1 -0.34375 0.28125 1 1 -0.34375 0.28125 1 1 -0.40625 0.28125 1 1 -0.40625 0.28125 1 1 -0.46875 0.28125 1 1 -0.46875 0.28125 1 1 -0.53125 0.28125 1 1 -0.53125 0.28125 1 1 -0.59375 0.28125 1 1 -0.59375 0.28125 1 1 -0.65625 0.28125 1 1 -0.65625 0.28125 1 1 -0.71875 0.28125 1 1 -0.71875 0.28125 1 1 -0.78125 0.28125 1 1 -0.78125 0.28125 1 1 -0.84375 0.28125 1 1 -0.84375 0.28125 1 1 -0.90625 0.28125 1 1 -0.90625 0.28125 1 1 -0.96875 0.28125 1 1 -0.96875 0.28125 1 1 -0.03125 0.34375 1 1 -0.03125 0.34375 1 1 -0.09375 0.34375 1 1 -0.09375 0.34375 1 1 -0.15625 0.34375 1 1 -0.15625 0.34375 1 1 -0.21875 0.34375 1 1 -0.21875 0.34375 1 1 -0.28125 0.34375 1 1 -0.28125 0.34375 1 1 -0.34375 0.34375 1 1 -0.34375 0.34375 1 1 -0.40625 0.34375 1 1 -0.40625 0.34375 1 1 -0.46875 0.34375 1 1 -0.46875 0.34375 1 1 -0.53125 0.34375 1 1 -0.53125 0.34375 1 1 -0.59375 0.34375 1 1 -0.59375 0.34375 1 1 -0.65625 0.34375 1 1 -0.65625 0.34375 1 1 -0.71875 0.34375 1 1 -0.71875 0.34375 1 1 -0.78125 0.34375 1 1 -0.78125 0.34375 1 1 -0.84375 0.34375 1 1 -0.84375 0.34375 1 1 -0.90625 0.34375 1 1 -0.90625 0.34375 1 1 -0.96875 0.34375 1 1 -0.96875 0.34375 1 1 -0.03125 0.40625 1 1 -0.03125 0.40625 1 1 -0.09375 0.40625 1 1 -0.09375 0.40625 1 1 -0.15625 0.40625 1 1 -0.15625 0.40625 1 1 -0.21875 0.40625 1 1 -0.21875 0.40625 1 1 -0.28125 0.40625 1 1 -0.28125 0.40625 1 1 -0.34375 0.40625 1 1 -0.34375 0.40625 1 1 -0.40625 0.40625 1 1 -0.40625 0.40625 1 1 -0.46875 0.40625 1 1 -0.46875 0.40625 1 1 -0.53125 0.40625 1 1 -0.53125 0.40625 1 1 -0.59375 0.40625 1 1 -0.59375 0.40625 1 1 -0.65625 0.40625 1 1 -0.65625 0.40625 1 1 -0.71875 0.40625 1 1 -0.71875 0.40625 1 1 -0.78125 0.40625 1 1 -0.78125 0.40625 1 1 -0.84375 0.40625 1 1 -0.84375 0.40625 1 1 -0.90625 0.40625 1 1 -0.90625 0.40625 1 1 -0.96875 0.40625 1 1 -0.96875 0.40625 1 1 -0.03125 0.46875 1 1 -0.03125 0.46875 1 1 -0.09375 0.46875 1 1 -0.09375 0.46875 1 1 -0.15625 0.46875 1 1 -0.15625 0.46875 1 1 -0.21875 0.46875 1 1 -0.21875 0.46875 1 1 -0.28125 0.46875 1 1 -0.28125 0.46875 1 1 -0.34375 0.46875 1 1 -0.34375 0.46875 1 1 -0.40625 0.46875 1 1 -0.40625 0.46875 1 1 -0.46875 0.46875 1 1 -0.46875 0.46875 1 1 -0.53125 0.46875 1 1 -0.53125 0.46875 1 1 -0.59375 0.46875 1 1 -0.59375 0.46875 1 1 -0.65625 0.46875 1 1 -0.65625 0.46875 1 1 -0.71875 0.46875 1 1 -0.71875 0.46875 1 1 -0.78125 0.46875 1 1 -0.78125 0.46875 1 1 -0.84375 0.46875 1 1 -0.84375 0.46875 1 1 -0.90625 0.46875 1 1 -0.90625 0.46875 1 1 -0.96875 0.46875 1 1 -0.96875 0.46875 1 1 -0.03125 0.53125 1 1 -0.03125 0.53125 1 1 -0.09375 0.53125 1 1 -0.09375 0.53125 1 1 -0.15625 0.53125 1 1 -0.15625 0.53125 1 1 -0.21875 0.53125 1 1 -0.21875 0.53125 1 1 -0.28125 0.53125 1 1 -0.28125 0.53125 1 1 -0.34375 0.53125 1 1 -0.34375 0.53125 1 1 -0.40625 0.53125 1 1 -0.40625 0.53125 1 1 -0.46875 0.53125 1 1 -0.46875 0.53125 1 1 -0.53125 0.53125 1 1 -0.53125 0.53125 1 1 -0.59375 0.53125 1 1 -0.59375 0.53125 1 1 -0.65625 0.53125 1 1 -0.65625 0.53125 1 1 -0.71875 0.53125 1 1 -0.71875 0.53125 1 1 -0.78125 0.53125 1 1 -0.78125 0.53125 1 1 -0.84375 0.53125 1 1 -0.84375 0.53125 1 1 -0.90625 0.53125 1 1 -0.90625 0.53125 1 1 -0.96875 0.53125 1 1 -0.96875 0.53125 1 1 -0.03125 0.59375 1 1 -0.03125 0.59375 1 1 -0.09375 0.59375 1 1 -0.09375 0.59375 1 1 -0.15625 0.59375 1 1 -0.15625 0.59375 1 1 -0.21875 0.59375 1 1 -0.21875 0.59375 1 1 -0.28125 0.59375 1 1 -0.28125 0.59375 1 1 -0.34375 0.59375 1 1 -0.34375 0.59375 1 1 -0.40625 0.59375 1 1 -0.40625 0.59375 1 1 -0.46875 0.59375 1 1 -0.46875 0.59375 1 1 -0.53125 0.59375 1 1 -0.53125 0.59375 1 1 -0.59375 0.59375 1 1 -0.59375 0.59375 1 1 -0.65625 0.59375 1 1 -0.65625 0.59375 1 1 -0.71875 0.59375 1 1 -0.71875 0.59375 1 1 -0.78125 0.59375 1 1 -0.78125 0.59375 1 1 -0.84375 0.59375 1 1 -0.84375 0.59375 1 1 -0.90625 0.59375 1 1 -0.90625 0.59375 1 1 -0.96875 0.59375 1 1 -0.96875 0.59375 1 1 -0.03125 0.65625 1 1 -0.03125 0.65625 1 1 -0.09375 0.65625 1 1 -0.09375 0.65625 1 1 -0.15625 0.65625 1 1 -0.15625 0.65625 1 1 -0.21875 0.65625 1 1 -0.21875 0.65625 1 1 -0.28125 0.65625 1 1 -0.28125 0.65625 1 1 -0.34375 0.65625 1 1 -0.34375 0.65625 1 1 -0.40625 0.65625 1 1 -0.40625 0.65625 1 1 -0.46875 0.65625 1 1 -0.46875 0.65625 1 1 -0.53125 0.65625 1 1 -0.53125 0.65625 1 1 -0.59375 0.65625 1 1 -0.59375 0.65625 1 1 -0.65625 0.65625 1 1 -0.65625 0.65625 1 1 -0.71875 0.65625 1 1 -0.71875 0.65625 1 1 -0.78125 0.65625 1 1 -0.78125 0.65625 1 1 -0.84375 0.65625 1 1 -0.84375 0.65625 1 1 -0.90625 0.65625 1 1 -0.90625 0.65625 1 1 -0.96875 0.65625 1 1 -0.96875 0.65625 1 1 -0.03125 0.71875 1 1 -0.03125 0.71875 1 1 -0.09375 0.71875 1 1 -0.09375 0.71875 1 1 -0.15625 0.71875 1 1 -0.15625 0.71875 1 1 -0.21875 0.71875 1 1 -0.21875 0.71875 1 1 -0.28125 0.71875 1 1 -0.28125 0.71875 1 1 -0.34375 0.71875 1 1 -0.34375 0.71875 1 1 -0.40625 0.71875 1 1 -0.40625 0.71875 1 1 -0.46875 0.71875 1 1 -0.46875 0.71875 1 1 -0.53125 0.71875 1 1 -0.53125 0.71875 1 1 -0.59375 0.71875 1 1 -0.59375 0.71875 1 1 -0.65625 0.71875 1 1 -0.65625 0.71875 1 1 -0.71875 0.71875 1 1 -0.71875 0.71875 1 1 -0.78125 0.71875 1 1 -0.78125 0.71875 1 1 -0.84375 0.71875 1 1 -0.84375 0.71875 1 1 -0.90625 0.71875 1 1 -0.90625 0.71875 1 1 -0.96875 0.71875 1 1 -0.96875 0.71875 1 1 -0.03125 0.78125 1 1 -0.03125 0.78125 1 1 -0.09375 0.78125 1 1 -0.09375 0.78125 1 1 -0.15625 0.78125 1 1 -0.15625 0.78125 1 1 -0.21875 0.78125 1 1 -0.21875 0.78125 1 1 -0.28125 0.78125 1 1 -0.28125 0.78125 1 1 -0.34375 0.78125 1 1 -0.34375 0.78125 1 1 -0.40625 0.78125 1 1 -0.40625 0.78125 1 1 -0.46875 0.78125 1 1 -0.46875 0.78125 1 1 -0.53125 0.78125 1 1 -0.53125 0.78125 1 1 -0.59375 0.78125 1 1 -0.59375 0.78125 1 1 -0.65625 0.78125 1 1 -0.65625 0.78125 1 1 -0.71875 0.78125 1 1 -0.71875 0.78125 1 1 -0.78125 0.78125 1 1 -0.78125 0.78125 1 1 -0.84375 0.78125 1 1 -0.84375 0.78125 1 1 -0.90625 0.78125 1 1 -0.90625 0.78125 1 1 -0.96875 0.78125 1 1 -0.96875 0.78125 1 1 -0.03125 0.84375 1 1 -0.03125 0.84375 1 1 -0.09375 0.84375 1 1 -0.09375 0.84375 1 1 -0.15625 0.84375 1 1 -0.15625 0.84375 1 1 -0.21875 0.84375 1 1 -0.21875 0.84375 1 1 -0.28125 0.84375 1 1 -0.28125 0.84375 1 1 -0.34375 0.84375 1 1 -0.34375 0.84375 1 1 -0.40625 0.84375 1 1 -0.40625 0.84375 1 1 -0.46875 0.84375 1 1 -0.46875 0.84375 1 1 -0.53125 0.84375 1 1 -0.53125 0.84375 1 1 -0.59375 0.84375 1 1 -0.59375 0.84375 1 1 -0.65625 0.84375 1 1 -0.65625 0.84375 1 1 -0.71875 0.84375 1 1 -0.71875 0.84375 1 1 -0.78125 0.84375 1 1 -0.78125 0.84375 1 1 -0.84375 0.84375 1 1 -0.84375 0.84375 1 1 -0.90625 0.84375 1 1 -0.90625 0.84375 1 1 -0.96875 0.84375 1 1 -0.96875 0.84375 1 1 -0.03125 0.90625 1 1 -0.03125 0.90625 1 1 -0.09375 0.90625 1 1 -0.09375 0.90625 1 1 -0.15625 0.90625 1 1 -0.15625 0.90625 1 1 -0.21875 0.90625 1 1 -0.21875 0.90625 1 1 -0.28125 0.90625 1 1 -0.28125 0.90625 1 1 -0.34375 0.90625 1 1 -0.34375 0.90625 1 1 -0.40625 0.90625 1 1 -0.40625 0.90625 1 1 -0.46875 0.90625 1 1 -0.46875 0.90625 1 1 -0.53125 0.90625 1 1 -0.53125 0.90625 1 1 -0.59375 0.90625 1 1 -0.59375 0.90625 1 1 -0.65625 0.90625 1 1 -0.65625 0.90625 1 1 -0.71875 0.90625 1 1 -0.71875 0.90625 1 1 -0.78125 0.90625 1 1 -0.78125 0.90625 1 1 -0.84375 0.90625 1 1 -0.84375 0.90625 1 1 -0.90625 0.90625 1 1 -0.90625 0.90625 1 1 -0.96875 0.90625 1 1 -0.96875 0.90625 1 1 -0.03125 0.96875 1 1 -0.03125 0.96875 1 1 -0.09375 0.96875 1 1 -0.09375 0.96875 1 1 -0.15625 0.96875 1 1 -0.15625 0.96875 1 1 -0.21875 0.96875 1 1 -0.21875 0.96875 1 1 -0.28125 0.96875 1 1 -0.28125 0.96875 1 1 -0.34375 0.96875 1 1 -0.34375 0.96875 1 1 -0.40625 0.96875 1 1 -0.40625 0.96875 1 1 -0.46875 0.96875 1 1 -0.46875 0.96875 1 1 -0.53125 0.96875 1 1 -0.53125 0.96875 1 1 -0.59375 0.96875 1 1 -0.59375 0.96875 1 1 -0.65625 0.96875 1 1 -0.65625 0.96875 1 1 -0.71875 0.96875 1 1 -0.71875 0.96875 1 1 -0.78125 0.96875 1 1 -0.78125 0.96875 1 1 -0.84375 0.96875 1 1 -0.84375 0.96875 1 1 -0.90625 0.96875 1 1 -0.90625 0.96875 1 1 -0.96875 0.96875 1 1 -0.96875 0.96875 1 1 -0.0625 0.0625 1 1 -0.0625 0.0625 1 1 -0.0625 0.0625 1 1 -0.0625 0.0625 1 1 -0.0625 0.0625 1 1 -0.0625 0.0625 1 1 -0.1875 0.0625 1 1 -0.1875 0.0625 1 1 -0.1875 0.0625 1 1 -0.1875 0.0625 1 1 -0.1875 0.0625 1 1 -0.1875 0.0625 1 1 -0.3125 0.0625 1 1 -0.3125 0.0625 1 1 -0.3125 0.0625 1 1 -0.3125 0.0625 1 1 -0.3125 0.0625 1 1 -0.3125 0.0625 1 1 -0.4375 0.0625 1 1 -0.4375 0.0625 1 1 -0.4375 0.0625 1 1 -0.4375 0.0625 1 1 -0.4375 0.0625 1 1 -0.4375 0.0625 1 1 -0.5625 0.0625 1 1 -0.5625 0.0625 1 1 -0.5625 0.0625 1 1 -0.5625 0.0625 1 1 -0.5625 0.0625 1 1 -0.5625 0.0625 1 1 -0.6875 0.0625 1 1 -0.6875 0.0625 1 1 -0.6875 0.0625 1 1 -0.6875 0.0625 1 1 -0.6875 0.0625 1 1 -0.6875 0.0625 1 1 -0.8125 0.0625 1 1 -0.8125 0.0625 1 1 -0.8125 0.0625 1 1 -0.8125 0.0625 1 1 -0.8125 0.0625 1 1 -0.8125 0.0625 1 1 -0.9375 0.0625 1 1 -0.9375 0.0625 1 1 -0.9375 0.0625 1 1 -0.9375 0.0625 1 1 -0.9375 0.0625 1 1 -0.9375 0.0625 1 1 -0.0625 0.1875 1 1 -0.0625 0.1875 1 1 -0.0625 0.1875 1 1 -0.0625 0.1875 1 1 -0.0625 0.1875 1 1 -0.0625 0.1875 1 1 -0.1875 0.1875 1 1 -0.1875 0.1875 1 1 -0.1875 0.1875 1 1 -0.1875 0.1875 1 1 -0.1875 0.1875 1 1 -0.1875 0.1875 1 1 -0.3125 0.1875 1 1 -0.3125 0.1875 1 1 -0.3125 0.1875 1 1 -0.3125 0.1875 1 1 -0.3125 0.1875 1 1 -0.3125 0.1875 1 1 -0.4375 0.1875 1 1 -0.4375 0.1875 1 1 -0.4375 0.1875 1 1 -0.4375 0.1875 1 1 -0.4375 0.1875 1 1 -0.4375 0.1875 1 1 -0.5625 0.1875 1 1 -0.5625 0.1875 1 1 -0.5625 0.1875 1 1 -0.5625 0.1875 1 1 -0.5625 0.1875 1 1 -0.5625 0.1875 1 1 -0.6875 0.1875 1 1 -0.6875 0.1875 1 1 -0.6875 0.1875 1 1 -0.6875 0.1875 1 1 -0.6875 0.1875 1 1 -0.6875 0.1875 1 1 -0.8125 0.1875 1 1 -0.8125 0.1875 1 1 -0.8125 0.1875 1 1 -0.8125 0.1875 1 1 -0.8125 0.1875 1 1 -0.8125 0.1875 1 1 -0.9375 0.1875 1 1 -0.9375 0.1875 1 1 -0.9375 0.1875 1 1 -0.9375 0.1875 1 1 -0.9375 0.1875 1 1 -0.9375 0.1875 1 1 -0.0625 0.3125 1 1 -0.0625 0.3125 1 1 -0.0625 0.3125 1 1 -0.0625 0.3125 1 1 -0.0625 0.3125 1 1 -0.0625 0.3125 1 1 -0.1875 0.3125 1 1 -0.1875 0.3125 1 1 -0.1875 0.3125 1 1 -0.1875 0.3125 1 1 -0.1875 0.3125 1 1 -0.1875 0.3125 1 1 -0.3125 0.3125 1 1 -0.3125 0.3125 1 1 -0.3125 0.3125 1 1 -0.3125 0.3125 1 1 -0.3125 0.3125 1 1 -0.3125 0.3125 1 1 -0.4375 0.3125 1 1 -0.4375 0.3125 1 1 -0.4375 0.3125 1 1 -0.4375 0.3125 1 1 -0.4375 0.3125 1 1 -0.4375 0.3125 1 1 -0.5625 0.3125 1 1 -0.5625 0.3125 1 1 -0.5625 0.3125 1 1 -0.5625 0.3125 1 1 -0.5625 0.3125 1 1 -0.5625 0.3125 1 1 -0.6875 0.3125 1 1 -0.6875 0.3125 1 1 -0.6875 0.3125 1 1 -0.6875 0.3125 1 1 -0.6875 0.3125 1 1 -0.6875 0.3125 1 1 -0.8125 0.3125 1 1 -0.8125 0.3125 1 1 -0.8125 0.3125 1 1 -0.8125 0.3125 1 1 -0.8125 0.3125 1 1 -0.8125 0.3125 1 1 -0.9375 0.3125 1 1 -0.9375 0.3125 1 1 -0.9375 0.3125 1 1 -0.9375 0.3125 1 1 -0.9375 0.3125 1 1 -0.9375 0.3125 1 1 -0.0625 0.4375 1 1 -0.0625 0.4375 1 1 -0.0625 0.4375 1 1 -0.0625 0.4375 1 1 -0.0625 0.4375 1 1 -0.0625 0.4375 1 1 -0.1875 0.4375 1 1 -0.1875 0.4375 1 1 -0.1875 0.4375 1 1 -0.1875 0.4375 1 1 -0.1875 0.4375 1 1 -0.1875 0.4375 1 1 -0.3125 0.4375 1 1 -0.3125 0.4375 1 1 -0.3125 0.4375 1 1 -0.3125 0.4375 1 1 -0.3125 0.4375 1 1 -0.3125 0.4375 1 1 -0.4375 0.4375 1 1 -0.4375 0.4375 1 1 -0.4375 0.4375 1 1 -0.4375 0.4375 1 1 -0.4375 0.4375 1 1 -0.4375 0.4375 1 1 -0.5625 0.4375 1 1 -0.5625 0.4375 1 1 -0.5625 0.4375 1 1 -0.5625 0.4375 1 1 -0.5625 0.4375 1 1 -0.5625 0.4375 1 1 -0.6875 0.4375 1 1 -0.6875 0.4375 1 1 -0.6875 0.4375 1 1 -0.6875 0.4375 1 1 -0.6875 0.4375 1 1 -0.6875 0.4375 1 1 -0.8125 0.4375 1 1 -0.8125 0.4375 1 1 -0.8125 0.4375 1 1 -0.8125 0.4375 1 1 -0.8125 0.4375 1 1 -0.8125 0.4375 1 1 -0.9375 0.4375 1 1 -0.9375 0.4375 1 1 -0.9375 0.4375 1 1 -0.9375 0.4375 1 1 -0.9375 0.4375 1 1 -0.9375 0.4375 1 1 -0.0625 0.5625 1 1 -0.0625 0.5625 1 1 -0.0625 0.5625 1 1 -0.0625 0.5625 1 1 -0.0625 0.5625 1 1 -0.0625 0.5625 1 1 -0.1875 0.5625 1 1 -0.1875 0.5625 1 1 -0.1875 0.5625 1 1 -0.1875 0.5625 1 1 -0.1875 0.5625 1 1 -0.1875 0.5625 1 1 -0.3125 0.5625 1 1 -0.3125 0.5625 1 1 -0.3125 0.5625 1 1 -0.3125 0.5625 1 1 -0.3125 0.5625 1 1 -0.3125 0.5625 1 1 -0.4375 0.5625 1 1 -0.4375 0.5625 1 1 -0.4375 0.5625 1 1 -0.4375 0.5625 1 1 -0.4375 0.5625 1 1 -0.4375 0.5625 1 1 -0.5625 0.5625 1 1 -0.5625 0.5625 1 1 -0.5625 0.5625 1 1 -0.5625 0.5625 1 1 -0.5625 0.5625 1 1 -0.5625 0.5625 1 1 -0.6875 0.5625 1 1 -0.6875 0.5625 1 1 -0.6875 0.5625 1 1 -0.6875 0.5625 1 1 -0.6875 0.5625 1 1 -0.6875 0.5625 1 1 -0.8125 0.5625 1 1 -0.8125 0.5625 1 1 -0.8125 0.5625 1 1 -0.8125 0.5625 1 1 -0.8125 0.5625 1 1 -0.8125 0.5625 1 1 -0.9375 0.5625 1 1 -0.9375 0.5625 1 1 -0.9375 0.5625 1 1 -0.9375 0.5625 1 1 -0.9375 0.5625 1 1 -0.9375 0.5625 1 1 -0.0625 0.6875 1 1 -0.0625 0.6875 1 1 -0.0625 0.6875 1 1 -0.0625 0.6875 1 1 -0.0625 0.6875 1 1 -0.0625 0.6875 1 1 -0.1875 0.6875 1 1 -0.1875 0.6875 1 1 -0.1875 0.6875 1 1 -0.1875 0.6875 1 1 -0.1875 0.6875 1 1 -0.1875 0.6875 1 1 -0.3125 0.6875 1 1 -0.3125 0.6875 1 1 -0.3125 0.6875 1 1 -0.3125 0.6875 1 1 -0.3125 0.6875 1 1 -0.3125 0.6875 1 1 -0.4375 0.6875 1 1 -0.4375 0.6875 1 1 -0.4375 0.6875 1 1 -0.4375 0.6875 1 1 -0.4375 0.6875 1 1 -0.4375 0.6875 1 1 -0.5625 0.6875 1 1 -0.5625 0.6875 1 1 -0.5625 0.6875 1 1 -0.5625 0.6875 1 1 -0.5625 0.6875 1 1 -0.5625 0.6875 1 1 -0.6875 0.6875 1 1 -0.6875 0.6875 1 1 -0.6875 0.6875 1 1 -0.6875 0.6875 1 1 -0.6875 0.6875 1 1 -0.6875 0.6875 1 1 -0.8125 0.6875 1 1 -0.8125 0.6875 1 1 -0.8125 0.6875 1 1 -0.8125 0.6875 1 1 -0.8125 0.6875 1 1 -0.8125 0.6875 1 1 -0.9375 0.6875 1 1 -0.9375 0.6875 1 1 -0.9375 0.6875 1 1 -0.9375 0.6875 1 1 -0.9375 0.6875 1 1 -0.9375 0.6875 1 1 -0.0625 0.8125 1 1 -0.0625 0.8125 1 1 -0.0625 0.8125 1 1 -0.0625 0.8125 1 1 -0.0625 0.8125 1 1 -0.0625 0.8125 1 1 -0.1875 0.8125 1 1 -0.1875 0.8125 1 1 -0.1875 0.8125 1 1 -0.1875 0.8125 1 1 -0.1875 0.8125 1 1 -0.1875 0.8125 1 1 -0.3125 0.8125 1 1 -0.3125 0.8125 1 1 -0.3125 0.8125 1 1 -0.3125 0.8125 1 1 -0.3125 0.8125 1 1 -0.3125 0.8125 1 1 -0.4375 0.8125 1 1 -0.4375 0.8125 1 1 -0.4375 0.8125 1 1 -0.4375 0.8125 1 1 -0.4375 0.8125 1 1 -0.4375 0.8125 1 1 -0.5625 0.8125 1 1 -0.5625 0.8125 1 1 -0.5625 0.8125 1 1 -0.5625 0.8125 1 1 -0.5625 0.8125 1 1 -0.5625 0.8125 1 1 -0.6875 0.8125 1 1 -0.6875 0.8125 1 1 -0.6875 0.8125 1 1 -0.6875 0.8125 1 1 -0.6875 0.8125 1 1 -0.6875 0.8125 1 1 -0.8125 0.8125 1 1 -0.8125 0.8125 1 1 -0.8125 0.8125 1 1 -0.8125 0.8125 1 1 -0.8125 0.8125 1 1 -0.8125 0.8125 1 1 -0.9375 0.8125 1 1 -0.9375 0.8125 1 1 -0.9375 0.8125 1 1 -0.9375 0.8125 1 1 -0.9375 0.8125 1 1 -0.9375 0.8125 1 1 -0.0625 0.9375 1 1 -0.0625 0.9375 1 1 -0.0625 0.9375 1 1 -0.0625 0.9375 1 1 -0.0625 0.9375 1 1 -0.0625 0.9375 1 1 -0.1875 0.9375 1 1 -0.1875 0.9375 1 1 -0.1875 0.9375 1 1 -0.1875 0.9375 1 1 -0.1875 0.9375 1 1 -0.1875 0.9375 1 1 -0.3125 0.9375 1 1 -0.3125 0.9375 1 1 -0.3125 0.9375 1 1 -0.3125 0.9375 1 1 -0.3125 0.9375 1 1 -0.3125 0.9375 1 1 -0.4375 0.9375 1 1 -0.4375 0.9375 1 1 -0.4375 0.9375 1 1 -0.4375 0.9375 1 1 -0.4375 0.9375 1 1 -0.4375 0.9375 1 1 -0.5625 0.9375 1 1 -0.5625 0.9375 1 1 -0.5625 0.9375 1 1 -0.5625 0.9375 1 1 -0.5625 0.9375 1 1 -0.5625 0.9375 1 1 -0.6875 0.9375 1 1 -0.6875 0.9375 1 1 -0.6875 0.9375 1 1 -0.6875 0.9375 1 1 -0.6875 0.9375 1 1 -0.6875 0.9375 1 1 -0.8125 0.9375 1 1 -0.8125 0.9375 1 1 -0.8125 0.9375 1 1 -0.8125 0.9375 1 1 -0.8125 0.9375 1 1 -0.8125 0.9375 1 1 -0.9375 0.9375 1 1 -0.9375 0.9375 1 1 -0.9375 0.9375 1 1 -0.9375 0.9375 1 1 -0.9375 0.9375 1 1 -0.9375 0.9375 1 1 diff --git a/mediapipe/calculators/tflite/testdata/anchor_golden_file_1.txt b/mediapipe/calculators/tflite/testdata/anchor_golden_file_1.txt deleted file mode 100644 index c894e0f8d..000000000 --- a/mediapipe/calculators/tflite/testdata/anchor_golden_file_1.txt +++ /dev/null @@ -1,1917 +0,0 @@ -0.0263158 0.0263158 0.1 0.1 -0.0263158 0.0263158 0.282843 0.141421 -0.0263158 0.0263158 0.141421 0.282843 -0.0789474 0.0263158 0.1 0.1 -0.0789474 0.0263158 0.282843 0.141421 -0.0789474 0.0263158 0.141421 0.282843 -0.131579 0.0263158 0.1 0.1 -0.131579 0.0263158 0.282843 0.141421 -0.131579 0.0263158 0.141421 0.282843 -0.184211 0.0263158 0.1 0.1 -0.184211 0.0263158 0.282843 0.141421 -0.184211 0.0263158 0.141421 0.282843 -0.236842 0.0263158 0.1 0.1 -0.236842 0.0263158 0.282843 0.141421 -0.236842 0.0263158 0.141421 0.282843 -0.289474 0.0263158 0.1 0.1 -0.289474 0.0263158 0.282843 0.141421 -0.289474 0.0263158 0.141421 0.282843 -0.342105 0.0263158 0.1 0.1 -0.342105 0.0263158 0.282843 0.141421 -0.342105 0.0263158 0.141421 0.282843 -0.394737 0.0263158 0.1 0.1 -0.394737 0.0263158 0.282843 0.141421 -0.394737 0.0263158 0.141421 0.282843 -0.447368 0.0263158 0.1 0.1 -0.447368 0.0263158 0.282843 0.141421 -0.447368 0.0263158 0.141421 0.282843 -0.5 0.0263158 0.1 0.1 -0.5 0.0263158 0.282843 0.141421 -0.5 0.0263158 0.141421 0.282843 -0.552632 0.0263158 0.1 0.1 -0.552632 0.0263158 0.282843 0.141421 -0.552632 0.0263158 0.141421 0.282843 -0.605263 0.0263158 0.1 0.1 -0.605263 0.0263158 0.282843 0.141421 -0.605263 0.0263158 0.141421 0.282843 -0.657895 0.0263158 0.1 0.1 -0.657895 0.0263158 0.282843 0.141421 -0.657895 0.0263158 0.141421 0.282843 -0.710526 0.0263158 0.1 0.1 -0.710526 0.0263158 0.282843 0.141421 -0.710526 0.0263158 0.141421 0.282843 -0.763158 0.0263158 0.1 0.1 -0.763158 0.0263158 0.282843 0.141421 -0.763158 0.0263158 0.141421 0.282843 -0.81579 0.0263158 0.1 0.1 -0.81579 0.0263158 0.282843 0.141421 -0.81579 0.0263158 0.141421 0.282843 -0.868421 0.0263158 0.1 0.1 -0.868421 0.0263158 0.282843 0.141421 -0.868421 0.0263158 0.141421 0.282843 -0.921053 0.0263158 0.1 0.1 -0.921053 0.0263158 0.282843 0.141421 -0.921053 0.0263158 0.141421 0.282843 -0.973684 0.0263158 0.1 0.1 -0.973684 0.0263158 0.282843 0.141421 -0.973684 0.0263158 0.141421 0.282843 -0.0263158 0.0789474 0.1 0.1 -0.0263158 0.0789474 0.282843 0.141421 -0.0263158 0.0789474 0.141421 0.282843 -0.0789474 0.0789474 0.1 0.1 -0.0789474 0.0789474 0.282843 0.141421 -0.0789474 0.0789474 0.141421 0.282843 -0.131579 0.0789474 0.1 0.1 -0.131579 0.0789474 0.282843 0.141421 -0.131579 0.0789474 0.141421 0.282843 -0.184211 0.0789474 0.1 0.1 -0.184211 0.0789474 0.282843 0.141421 -0.184211 0.0789474 0.141421 0.282843 -0.236842 0.0789474 0.1 0.1 -0.236842 0.0789474 0.282843 0.141421 -0.236842 0.0789474 0.141421 0.282843 -0.289474 0.0789474 0.1 0.1 -0.289474 0.0789474 0.282843 0.141421 -0.289474 0.0789474 0.141421 0.282843 -0.342105 0.0789474 0.1 0.1 -0.342105 0.0789474 0.282843 0.141421 -0.342105 0.0789474 0.141421 0.282843 -0.394737 0.0789474 0.1 0.1 -0.394737 0.0789474 0.282843 0.141421 -0.394737 0.0789474 0.141421 0.282843 -0.447368 0.0789474 0.1 0.1 -0.447368 0.0789474 0.282843 0.141421 -0.447368 0.0789474 0.141421 0.282843 -0.5 0.0789474 0.1 0.1 -0.5 0.0789474 0.282843 0.141421 -0.5 0.0789474 0.141421 0.282843 -0.552632 0.0789474 0.1 0.1 -0.552632 0.0789474 0.282843 0.141421 -0.552632 0.0789474 0.141421 0.282843 -0.605263 0.0789474 0.1 0.1 -0.605263 0.0789474 0.282843 0.141421 -0.605263 0.0789474 0.141421 0.282843 -0.657895 0.0789474 0.1 0.1 -0.657895 0.0789474 0.282843 0.141421 -0.657895 0.0789474 0.141421 0.282843 -0.710526 0.0789474 0.1 0.1 -0.710526 0.0789474 0.282843 0.141421 -0.710526 0.0789474 0.141421 0.282843 -0.763158 0.0789474 0.1 0.1 -0.763158 0.0789474 0.282843 0.141421 -0.763158 0.0789474 0.141421 0.282843 -0.81579 0.0789474 0.1 0.1 -0.81579 0.0789474 0.282843 0.141421 -0.81579 0.0789474 0.141421 0.282843 -0.868421 0.0789474 0.1 0.1 -0.868421 0.0789474 0.282843 0.141421 -0.868421 0.0789474 0.141421 0.282843 -0.921053 0.0789474 0.1 0.1 -0.921053 0.0789474 0.282843 0.141421 -0.921053 0.0789474 0.141421 0.282843 -0.973684 0.0789474 0.1 0.1 -0.973684 0.0789474 0.282843 0.141421 -0.973684 0.0789474 0.141421 0.282843 -0.0263158 0.131579 0.1 0.1 -0.0263158 0.131579 0.282843 0.141421 -0.0263158 0.131579 0.141421 0.282843 -0.0789474 0.131579 0.1 0.1 -0.0789474 0.131579 0.282843 0.141421 -0.0789474 0.131579 0.141421 0.282843 -0.131579 0.131579 0.1 0.1 -0.131579 0.131579 0.282843 0.141421 -0.131579 0.131579 0.141421 0.282843 -0.184211 0.131579 0.1 0.1 -0.184211 0.131579 0.282843 0.141421 -0.184211 0.131579 0.141421 0.282843 -0.236842 0.131579 0.1 0.1 -0.236842 0.131579 0.282843 0.141421 -0.236842 0.131579 0.141421 0.282843 -0.289474 0.131579 0.1 0.1 -0.289474 0.131579 0.282843 0.141421 -0.289474 0.131579 0.141421 0.282843 -0.342105 0.131579 0.1 0.1 -0.342105 0.131579 0.282843 0.141421 -0.342105 0.131579 0.141421 0.282843 -0.394737 0.131579 0.1 0.1 -0.394737 0.131579 0.282843 0.141421 -0.394737 0.131579 0.141421 0.282843 -0.447368 0.131579 0.1 0.1 -0.447368 0.131579 0.282843 0.141421 -0.447368 0.131579 0.141421 0.282843 -0.5 0.131579 0.1 0.1 -0.5 0.131579 0.282843 0.141421 -0.5 0.131579 0.141421 0.282843 -0.552632 0.131579 0.1 0.1 -0.552632 0.131579 0.282843 0.141421 -0.552632 0.131579 0.141421 0.282843 -0.605263 0.131579 0.1 0.1 -0.605263 0.131579 0.282843 0.141421 -0.605263 0.131579 0.141421 0.282843 -0.657895 0.131579 0.1 0.1 -0.657895 0.131579 0.282843 0.141421 -0.657895 0.131579 0.141421 0.282843 -0.710526 0.131579 0.1 0.1 -0.710526 0.131579 0.282843 0.141421 -0.710526 0.131579 0.141421 0.282843 -0.763158 0.131579 0.1 0.1 -0.763158 0.131579 0.282843 0.141421 -0.763158 0.131579 0.141421 0.282843 -0.81579 0.131579 0.1 0.1 -0.81579 0.131579 0.282843 0.141421 -0.81579 0.131579 0.141421 0.282843 -0.868421 0.131579 0.1 0.1 -0.868421 0.131579 0.282843 0.141421 -0.868421 0.131579 0.141421 0.282843 -0.921053 0.131579 0.1 0.1 -0.921053 0.131579 0.282843 0.141421 -0.921053 0.131579 0.141421 0.282843 -0.973684 0.131579 0.1 0.1 -0.973684 0.131579 0.282843 0.141421 -0.973684 0.131579 0.141421 0.282843 -0.0263158 0.184211 0.1 0.1 -0.0263158 0.184211 0.282843 0.141421 -0.0263158 0.184211 0.141421 0.282843 -0.0789474 0.184211 0.1 0.1 -0.0789474 0.184211 0.282843 0.141421 -0.0789474 0.184211 0.141421 0.282843 -0.131579 0.184211 0.1 0.1 -0.131579 0.184211 0.282843 0.141421 -0.131579 0.184211 0.141421 0.282843 -0.184211 0.184211 0.1 0.1 -0.184211 0.184211 0.282843 0.141421 -0.184211 0.184211 0.141421 0.282843 -0.236842 0.184211 0.1 0.1 -0.236842 0.184211 0.282843 0.141421 -0.236842 0.184211 0.141421 0.282843 -0.289474 0.184211 0.1 0.1 -0.289474 0.184211 0.282843 0.141421 -0.289474 0.184211 0.141421 0.282843 -0.342105 0.184211 0.1 0.1 -0.342105 0.184211 0.282843 0.141421 -0.342105 0.184211 0.141421 0.282843 -0.394737 0.184211 0.1 0.1 -0.394737 0.184211 0.282843 0.141421 -0.394737 0.184211 0.141421 0.282843 -0.447368 0.184211 0.1 0.1 -0.447368 0.184211 0.282843 0.141421 -0.447368 0.184211 0.141421 0.282843 -0.5 0.184211 0.1 0.1 -0.5 0.184211 0.282843 0.141421 -0.5 0.184211 0.141421 0.282843 -0.552632 0.184211 0.1 0.1 -0.552632 0.184211 0.282843 0.141421 -0.552632 0.184211 0.141421 0.282843 -0.605263 0.184211 0.1 0.1 -0.605263 0.184211 0.282843 0.141421 -0.605263 0.184211 0.141421 0.282843 -0.657895 0.184211 0.1 0.1 -0.657895 0.184211 0.282843 0.141421 -0.657895 0.184211 0.141421 0.282843 -0.710526 0.184211 0.1 0.1 -0.710526 0.184211 0.282843 0.141421 -0.710526 0.184211 0.141421 0.282843 -0.763158 0.184211 0.1 0.1 -0.763158 0.184211 0.282843 0.141421 -0.763158 0.184211 0.141421 0.282843 -0.81579 0.184211 0.1 0.1 -0.81579 0.184211 0.282843 0.141421 -0.81579 0.184211 0.141421 0.282843 -0.868421 0.184211 0.1 0.1 -0.868421 0.184211 0.282843 0.141421 -0.868421 0.184211 0.141421 0.282843 -0.921053 0.184211 0.1 0.1 -0.921053 0.184211 0.282843 0.141421 -0.921053 0.184211 0.141421 0.282843 -0.973684 0.184211 0.1 0.1 -0.973684 0.184211 0.282843 0.141421 -0.973684 0.184211 0.141421 0.282843 -0.0263158 0.236842 0.1 0.1 -0.0263158 0.236842 0.282843 0.141421 -0.0263158 0.236842 0.141421 0.282843 -0.0789474 0.236842 0.1 0.1 -0.0789474 0.236842 0.282843 0.141421 -0.0789474 0.236842 0.141421 0.282843 -0.131579 0.236842 0.1 0.1 -0.131579 0.236842 0.282843 0.141421 -0.131579 0.236842 0.141421 0.282843 -0.184211 0.236842 0.1 0.1 -0.184211 0.236842 0.282843 0.141421 -0.184211 0.236842 0.141421 0.282843 -0.236842 0.236842 0.1 0.1 -0.236842 0.236842 0.282843 0.141421 -0.236842 0.236842 0.141421 0.282843 -0.289474 0.236842 0.1 0.1 -0.289474 0.236842 0.282843 0.141421 -0.289474 0.236842 0.141421 0.282843 -0.342105 0.236842 0.1 0.1 -0.342105 0.236842 0.282843 0.141421 -0.342105 0.236842 0.141421 0.282843 -0.394737 0.236842 0.1 0.1 -0.394737 0.236842 0.282843 0.141421 -0.394737 0.236842 0.141421 0.282843 -0.447368 0.236842 0.1 0.1 -0.447368 0.236842 0.282843 0.141421 -0.447368 0.236842 0.141421 0.282843 -0.5 0.236842 0.1 0.1 -0.5 0.236842 0.282843 0.141421 -0.5 0.236842 0.141421 0.282843 -0.552632 0.236842 0.1 0.1 -0.552632 0.236842 0.282843 0.141421 -0.552632 0.236842 0.141421 0.282843 -0.605263 0.236842 0.1 0.1 -0.605263 0.236842 0.282843 0.141421 -0.605263 0.236842 0.141421 0.282843 -0.657895 0.236842 0.1 0.1 -0.657895 0.236842 0.282843 0.141421 -0.657895 0.236842 0.141421 0.282843 -0.710526 0.236842 0.1 0.1 -0.710526 0.236842 0.282843 0.141421 -0.710526 0.236842 0.141421 0.282843 -0.763158 0.236842 0.1 0.1 -0.763158 0.236842 0.282843 0.141421 -0.763158 0.236842 0.141421 0.282843 -0.81579 0.236842 0.1 0.1 -0.81579 0.236842 0.282843 0.141421 -0.81579 0.236842 0.141421 0.282843 -0.868421 0.236842 0.1 0.1 -0.868421 0.236842 0.282843 0.141421 -0.868421 0.236842 0.141421 0.282843 -0.921053 0.236842 0.1 0.1 -0.921053 0.236842 0.282843 0.141421 -0.921053 0.236842 0.141421 0.282843 -0.973684 0.236842 0.1 0.1 -0.973684 0.236842 0.282843 0.141421 -0.973684 0.236842 0.141421 0.282843 -0.0263158 0.289474 0.1 0.1 -0.0263158 0.289474 0.282843 0.141421 -0.0263158 0.289474 0.141421 0.282843 -0.0789474 0.289474 0.1 0.1 -0.0789474 0.289474 0.282843 0.141421 -0.0789474 0.289474 0.141421 0.282843 -0.131579 0.289474 0.1 0.1 -0.131579 0.289474 0.282843 0.141421 -0.131579 0.289474 0.141421 0.282843 -0.184211 0.289474 0.1 0.1 -0.184211 0.289474 0.282843 0.141421 -0.184211 0.289474 0.141421 0.282843 -0.236842 0.289474 0.1 0.1 -0.236842 0.289474 0.282843 0.141421 -0.236842 0.289474 0.141421 0.282843 -0.289474 0.289474 0.1 0.1 -0.289474 0.289474 0.282843 0.141421 -0.289474 0.289474 0.141421 0.282843 -0.342105 0.289474 0.1 0.1 -0.342105 0.289474 0.282843 0.141421 -0.342105 0.289474 0.141421 0.282843 -0.394737 0.289474 0.1 0.1 -0.394737 0.289474 0.282843 0.141421 -0.394737 0.289474 0.141421 0.282843 -0.447368 0.289474 0.1 0.1 -0.447368 0.289474 0.282843 0.141421 -0.447368 0.289474 0.141421 0.282843 -0.5 0.289474 0.1 0.1 -0.5 0.289474 0.282843 0.141421 -0.5 0.289474 0.141421 0.282843 -0.552632 0.289474 0.1 0.1 -0.552632 0.289474 0.282843 0.141421 -0.552632 0.289474 0.141421 0.282843 -0.605263 0.289474 0.1 0.1 -0.605263 0.289474 0.282843 0.141421 -0.605263 0.289474 0.141421 0.282843 -0.657895 0.289474 0.1 0.1 -0.657895 0.289474 0.282843 0.141421 -0.657895 0.289474 0.141421 0.282843 -0.710526 0.289474 0.1 0.1 -0.710526 0.289474 0.282843 0.141421 -0.710526 0.289474 0.141421 0.282843 -0.763158 0.289474 0.1 0.1 -0.763158 0.289474 0.282843 0.141421 -0.763158 0.289474 0.141421 0.282843 -0.81579 0.289474 0.1 0.1 -0.81579 0.289474 0.282843 0.141421 -0.81579 0.289474 0.141421 0.282843 -0.868421 0.289474 0.1 0.1 -0.868421 0.289474 0.282843 0.141421 -0.868421 0.289474 0.141421 0.282843 -0.921053 0.289474 0.1 0.1 -0.921053 0.289474 0.282843 0.141421 -0.921053 0.289474 0.141421 0.282843 -0.973684 0.289474 0.1 0.1 -0.973684 0.289474 0.282843 0.141421 -0.973684 0.289474 0.141421 0.282843 -0.0263158 0.342105 0.1 0.1 -0.0263158 0.342105 0.282843 0.141421 -0.0263158 0.342105 0.141421 0.282843 -0.0789474 0.342105 0.1 0.1 -0.0789474 0.342105 0.282843 0.141421 -0.0789474 0.342105 0.141421 0.282843 -0.131579 0.342105 0.1 0.1 -0.131579 0.342105 0.282843 0.141421 -0.131579 0.342105 0.141421 0.282843 -0.184211 0.342105 0.1 0.1 -0.184211 0.342105 0.282843 0.141421 -0.184211 0.342105 0.141421 0.282843 -0.236842 0.342105 0.1 0.1 -0.236842 0.342105 0.282843 0.141421 -0.236842 0.342105 0.141421 0.282843 -0.289474 0.342105 0.1 0.1 -0.289474 0.342105 0.282843 0.141421 -0.289474 0.342105 0.141421 0.282843 -0.342105 0.342105 0.1 0.1 -0.342105 0.342105 0.282843 0.141421 -0.342105 0.342105 0.141421 0.282843 -0.394737 0.342105 0.1 0.1 -0.394737 0.342105 0.282843 0.141421 -0.394737 0.342105 0.141421 0.282843 -0.447368 0.342105 0.1 0.1 -0.447368 0.342105 0.282843 0.141421 -0.447368 0.342105 0.141421 0.282843 -0.5 0.342105 0.1 0.1 -0.5 0.342105 0.282843 0.141421 -0.5 0.342105 0.141421 0.282843 -0.552632 0.342105 0.1 0.1 -0.552632 0.342105 0.282843 0.141421 -0.552632 0.342105 0.141421 0.282843 -0.605263 0.342105 0.1 0.1 -0.605263 0.342105 0.282843 0.141421 -0.605263 0.342105 0.141421 0.282843 -0.657895 0.342105 0.1 0.1 -0.657895 0.342105 0.282843 0.141421 -0.657895 0.342105 0.141421 0.282843 -0.710526 0.342105 0.1 0.1 -0.710526 0.342105 0.282843 0.141421 -0.710526 0.342105 0.141421 0.282843 -0.763158 0.342105 0.1 0.1 -0.763158 0.342105 0.282843 0.141421 -0.763158 0.342105 0.141421 0.282843 -0.81579 0.342105 0.1 0.1 -0.81579 0.342105 0.282843 0.141421 -0.81579 0.342105 0.141421 0.282843 -0.868421 0.342105 0.1 0.1 -0.868421 0.342105 0.282843 0.141421 -0.868421 0.342105 0.141421 0.282843 -0.921053 0.342105 0.1 0.1 -0.921053 0.342105 0.282843 0.141421 -0.921053 0.342105 0.141421 0.282843 -0.973684 0.342105 0.1 0.1 -0.973684 0.342105 0.282843 0.141421 -0.973684 0.342105 0.141421 0.282843 -0.0263158 0.394737 0.1 0.1 -0.0263158 0.394737 0.282843 0.141421 -0.0263158 0.394737 0.141421 0.282843 -0.0789474 0.394737 0.1 0.1 -0.0789474 0.394737 0.282843 0.141421 -0.0789474 0.394737 0.141421 0.282843 -0.131579 0.394737 0.1 0.1 -0.131579 0.394737 0.282843 0.141421 -0.131579 0.394737 0.141421 0.282843 -0.184211 0.394737 0.1 0.1 -0.184211 0.394737 0.282843 0.141421 -0.184211 0.394737 0.141421 0.282843 -0.236842 0.394737 0.1 0.1 -0.236842 0.394737 0.282843 0.141421 -0.236842 0.394737 0.141421 0.282843 -0.289474 0.394737 0.1 0.1 -0.289474 0.394737 0.282843 0.141421 -0.289474 0.394737 0.141421 0.282843 -0.342105 0.394737 0.1 0.1 -0.342105 0.394737 0.282843 0.141421 -0.342105 0.394737 0.141421 0.282843 -0.394737 0.394737 0.1 0.1 -0.394737 0.394737 0.282843 0.141421 -0.394737 0.394737 0.141421 0.282843 -0.447368 0.394737 0.1 0.1 -0.447368 0.394737 0.282843 0.141421 -0.447368 0.394737 0.141421 0.282843 -0.5 0.394737 0.1 0.1 -0.5 0.394737 0.282843 0.141421 -0.5 0.394737 0.141421 0.282843 -0.552632 0.394737 0.1 0.1 -0.552632 0.394737 0.282843 0.141421 -0.552632 0.394737 0.141421 0.282843 -0.605263 0.394737 0.1 0.1 -0.605263 0.394737 0.282843 0.141421 -0.605263 0.394737 0.141421 0.282843 -0.657895 0.394737 0.1 0.1 -0.657895 0.394737 0.282843 0.141421 -0.657895 0.394737 0.141421 0.282843 -0.710526 0.394737 0.1 0.1 -0.710526 0.394737 0.282843 0.141421 -0.710526 0.394737 0.141421 0.282843 -0.763158 0.394737 0.1 0.1 -0.763158 0.394737 0.282843 0.141421 -0.763158 0.394737 0.141421 0.282843 -0.81579 0.394737 0.1 0.1 -0.81579 0.394737 0.282843 0.141421 -0.81579 0.394737 0.141421 0.282843 -0.868421 0.394737 0.1 0.1 -0.868421 0.394737 0.282843 0.141421 -0.868421 0.394737 0.141421 0.282843 -0.921053 0.394737 0.1 0.1 -0.921053 0.394737 0.282843 0.141421 -0.921053 0.394737 0.141421 0.282843 -0.973684 0.394737 0.1 0.1 -0.973684 0.394737 0.282843 0.141421 -0.973684 0.394737 0.141421 0.282843 -0.0263158 0.447368 0.1 0.1 -0.0263158 0.447368 0.282843 0.141421 -0.0263158 0.447368 0.141421 0.282843 -0.0789474 0.447368 0.1 0.1 -0.0789474 0.447368 0.282843 0.141421 -0.0789474 0.447368 0.141421 0.282843 -0.131579 0.447368 0.1 0.1 -0.131579 0.447368 0.282843 0.141421 -0.131579 0.447368 0.141421 0.282843 -0.184211 0.447368 0.1 0.1 -0.184211 0.447368 0.282843 0.141421 -0.184211 0.447368 0.141421 0.282843 -0.236842 0.447368 0.1 0.1 -0.236842 0.447368 0.282843 0.141421 -0.236842 0.447368 0.141421 0.282843 -0.289474 0.447368 0.1 0.1 -0.289474 0.447368 0.282843 0.141421 -0.289474 0.447368 0.141421 0.282843 -0.342105 0.447368 0.1 0.1 -0.342105 0.447368 0.282843 0.141421 -0.342105 0.447368 0.141421 0.282843 -0.394737 0.447368 0.1 0.1 -0.394737 0.447368 0.282843 0.141421 -0.394737 0.447368 0.141421 0.282843 -0.447368 0.447368 0.1 0.1 -0.447368 0.447368 0.282843 0.141421 -0.447368 0.447368 0.141421 0.282843 -0.5 0.447368 0.1 0.1 -0.5 0.447368 0.282843 0.141421 -0.5 0.447368 0.141421 0.282843 -0.552632 0.447368 0.1 0.1 -0.552632 0.447368 0.282843 0.141421 -0.552632 0.447368 0.141421 0.282843 -0.605263 0.447368 0.1 0.1 -0.605263 0.447368 0.282843 0.141421 -0.605263 0.447368 0.141421 0.282843 -0.657895 0.447368 0.1 0.1 -0.657895 0.447368 0.282843 0.141421 -0.657895 0.447368 0.141421 0.282843 -0.710526 0.447368 0.1 0.1 -0.710526 0.447368 0.282843 0.141421 -0.710526 0.447368 0.141421 0.282843 -0.763158 0.447368 0.1 0.1 -0.763158 0.447368 0.282843 0.141421 -0.763158 0.447368 0.141421 0.282843 -0.81579 0.447368 0.1 0.1 -0.81579 0.447368 0.282843 0.141421 -0.81579 0.447368 0.141421 0.282843 -0.868421 0.447368 0.1 0.1 -0.868421 0.447368 0.282843 0.141421 -0.868421 0.447368 0.141421 0.282843 -0.921053 0.447368 0.1 0.1 -0.921053 0.447368 0.282843 0.141421 -0.921053 0.447368 0.141421 0.282843 -0.973684 0.447368 0.1 0.1 -0.973684 0.447368 0.282843 0.141421 -0.973684 0.447368 0.141421 0.282843 -0.0263158 0.5 0.1 0.1 -0.0263158 0.5 0.282843 0.141421 -0.0263158 0.5 0.141421 0.282843 -0.0789474 0.5 0.1 0.1 -0.0789474 0.5 0.282843 0.141421 -0.0789474 0.5 0.141421 0.282843 -0.131579 0.5 0.1 0.1 -0.131579 0.5 0.282843 0.141421 -0.131579 0.5 0.141421 0.282843 -0.184211 0.5 0.1 0.1 -0.184211 0.5 0.282843 0.141421 -0.184211 0.5 0.141421 0.282843 -0.236842 0.5 0.1 0.1 -0.236842 0.5 0.282843 0.141421 -0.236842 0.5 0.141421 0.282843 -0.289474 0.5 0.1 0.1 -0.289474 0.5 0.282843 0.141421 -0.289474 0.5 0.141421 0.282843 -0.342105 0.5 0.1 0.1 -0.342105 0.5 0.282843 0.141421 -0.342105 0.5 0.141421 0.282843 -0.394737 0.5 0.1 0.1 -0.394737 0.5 0.282843 0.141421 -0.394737 0.5 0.141421 0.282843 -0.447368 0.5 0.1 0.1 -0.447368 0.5 0.282843 0.141421 -0.447368 0.5 0.141421 0.282843 -0.5 0.5 0.1 0.1 -0.5 0.5 0.282843 0.141421 -0.5 0.5 0.141421 0.282843 -0.552632 0.5 0.1 0.1 -0.552632 0.5 0.282843 0.141421 -0.552632 0.5 0.141421 0.282843 -0.605263 0.5 0.1 0.1 -0.605263 0.5 0.282843 0.141421 -0.605263 0.5 0.141421 0.282843 -0.657895 0.5 0.1 0.1 -0.657895 0.5 0.282843 0.141421 -0.657895 0.5 0.141421 0.282843 -0.710526 0.5 0.1 0.1 -0.710526 0.5 0.282843 0.141421 -0.710526 0.5 0.141421 0.282843 -0.763158 0.5 0.1 0.1 -0.763158 0.5 0.282843 0.141421 -0.763158 0.5 0.141421 0.282843 -0.81579 0.5 0.1 0.1 -0.81579 0.5 0.282843 0.141421 -0.81579 0.5 0.141421 0.282843 -0.868421 0.5 0.1 0.1 -0.868421 0.5 0.282843 0.141421 -0.868421 0.5 0.141421 0.282843 -0.921053 0.5 0.1 0.1 -0.921053 0.5 0.282843 0.141421 -0.921053 0.5 0.141421 0.282843 -0.973684 0.5 0.1 0.1 -0.973684 0.5 0.282843 0.141421 -0.973684 0.5 0.141421 0.282843 -0.0263158 0.552632 0.1 0.1 -0.0263158 0.552632 0.282843 0.141421 -0.0263158 0.552632 0.141421 0.282843 -0.0789474 0.552632 0.1 0.1 -0.0789474 0.552632 0.282843 0.141421 -0.0789474 0.552632 0.141421 0.282843 -0.131579 0.552632 0.1 0.1 -0.131579 0.552632 0.282843 0.141421 -0.131579 0.552632 0.141421 0.282843 -0.184211 0.552632 0.1 0.1 -0.184211 0.552632 0.282843 0.141421 -0.184211 0.552632 0.141421 0.282843 -0.236842 0.552632 0.1 0.1 -0.236842 0.552632 0.282843 0.141421 -0.236842 0.552632 0.141421 0.282843 -0.289474 0.552632 0.1 0.1 -0.289474 0.552632 0.282843 0.141421 -0.289474 0.552632 0.141421 0.282843 -0.342105 0.552632 0.1 0.1 -0.342105 0.552632 0.282843 0.141421 -0.342105 0.552632 0.141421 0.282843 -0.394737 0.552632 0.1 0.1 -0.394737 0.552632 0.282843 0.141421 -0.394737 0.552632 0.141421 0.282843 -0.447368 0.552632 0.1 0.1 -0.447368 0.552632 0.282843 0.141421 -0.447368 0.552632 0.141421 0.282843 -0.5 0.552632 0.1 0.1 -0.5 0.552632 0.282843 0.141421 -0.5 0.552632 0.141421 0.282843 -0.552632 0.552632 0.1 0.1 -0.552632 0.552632 0.282843 0.141421 -0.552632 0.552632 0.141421 0.282843 -0.605263 0.552632 0.1 0.1 -0.605263 0.552632 0.282843 0.141421 -0.605263 0.552632 0.141421 0.282843 -0.657895 0.552632 0.1 0.1 -0.657895 0.552632 0.282843 0.141421 -0.657895 0.552632 0.141421 0.282843 -0.710526 0.552632 0.1 0.1 -0.710526 0.552632 0.282843 0.141421 -0.710526 0.552632 0.141421 0.282843 -0.763158 0.552632 0.1 0.1 -0.763158 0.552632 0.282843 0.141421 -0.763158 0.552632 0.141421 0.282843 -0.81579 0.552632 0.1 0.1 -0.81579 0.552632 0.282843 0.141421 -0.81579 0.552632 0.141421 0.282843 -0.868421 0.552632 0.1 0.1 -0.868421 0.552632 0.282843 0.141421 -0.868421 0.552632 0.141421 0.282843 -0.921053 0.552632 0.1 0.1 -0.921053 0.552632 0.282843 0.141421 -0.921053 0.552632 0.141421 0.282843 -0.973684 0.552632 0.1 0.1 -0.973684 0.552632 0.282843 0.141421 -0.973684 0.552632 0.141421 0.282843 -0.0263158 0.605263 0.1 0.1 -0.0263158 0.605263 0.282843 0.141421 -0.0263158 0.605263 0.141421 0.282843 -0.0789474 0.605263 0.1 0.1 -0.0789474 0.605263 0.282843 0.141421 -0.0789474 0.605263 0.141421 0.282843 -0.131579 0.605263 0.1 0.1 -0.131579 0.605263 0.282843 0.141421 -0.131579 0.605263 0.141421 0.282843 -0.184211 0.605263 0.1 0.1 -0.184211 0.605263 0.282843 0.141421 -0.184211 0.605263 0.141421 0.282843 -0.236842 0.605263 0.1 0.1 -0.236842 0.605263 0.282843 0.141421 -0.236842 0.605263 0.141421 0.282843 -0.289474 0.605263 0.1 0.1 -0.289474 0.605263 0.282843 0.141421 -0.289474 0.605263 0.141421 0.282843 -0.342105 0.605263 0.1 0.1 -0.342105 0.605263 0.282843 0.141421 -0.342105 0.605263 0.141421 0.282843 -0.394737 0.605263 0.1 0.1 -0.394737 0.605263 0.282843 0.141421 -0.394737 0.605263 0.141421 0.282843 -0.447368 0.605263 0.1 0.1 -0.447368 0.605263 0.282843 0.141421 -0.447368 0.605263 0.141421 0.282843 -0.5 0.605263 0.1 0.1 -0.5 0.605263 0.282843 0.141421 -0.5 0.605263 0.141421 0.282843 -0.552632 0.605263 0.1 0.1 -0.552632 0.605263 0.282843 0.141421 -0.552632 0.605263 0.141421 0.282843 -0.605263 0.605263 0.1 0.1 -0.605263 0.605263 0.282843 0.141421 -0.605263 0.605263 0.141421 0.282843 -0.657895 0.605263 0.1 0.1 -0.657895 0.605263 0.282843 0.141421 -0.657895 0.605263 0.141421 0.282843 -0.710526 0.605263 0.1 0.1 -0.710526 0.605263 0.282843 0.141421 -0.710526 0.605263 0.141421 0.282843 -0.763158 0.605263 0.1 0.1 -0.763158 0.605263 0.282843 0.141421 -0.763158 0.605263 0.141421 0.282843 -0.81579 0.605263 0.1 0.1 -0.81579 0.605263 0.282843 0.141421 -0.81579 0.605263 0.141421 0.282843 -0.868421 0.605263 0.1 0.1 -0.868421 0.605263 0.282843 0.141421 -0.868421 0.605263 0.141421 0.282843 -0.921053 0.605263 0.1 0.1 -0.921053 0.605263 0.282843 0.141421 -0.921053 0.605263 0.141421 0.282843 -0.973684 0.605263 0.1 0.1 -0.973684 0.605263 0.282843 0.141421 -0.973684 0.605263 0.141421 0.282843 -0.0263158 0.657895 0.1 0.1 -0.0263158 0.657895 0.282843 0.141421 -0.0263158 0.657895 0.141421 0.282843 -0.0789474 0.657895 0.1 0.1 -0.0789474 0.657895 0.282843 0.141421 -0.0789474 0.657895 0.141421 0.282843 -0.131579 0.657895 0.1 0.1 -0.131579 0.657895 0.282843 0.141421 -0.131579 0.657895 0.141421 0.282843 -0.184211 0.657895 0.1 0.1 -0.184211 0.657895 0.282843 0.141421 -0.184211 0.657895 0.141421 0.282843 -0.236842 0.657895 0.1 0.1 -0.236842 0.657895 0.282843 0.141421 -0.236842 0.657895 0.141421 0.282843 -0.289474 0.657895 0.1 0.1 -0.289474 0.657895 0.282843 0.141421 -0.289474 0.657895 0.141421 0.282843 -0.342105 0.657895 0.1 0.1 -0.342105 0.657895 0.282843 0.141421 -0.342105 0.657895 0.141421 0.282843 -0.394737 0.657895 0.1 0.1 -0.394737 0.657895 0.282843 0.141421 -0.394737 0.657895 0.141421 0.282843 -0.447368 0.657895 0.1 0.1 -0.447368 0.657895 0.282843 0.141421 -0.447368 0.657895 0.141421 0.282843 -0.5 0.657895 0.1 0.1 -0.5 0.657895 0.282843 0.141421 -0.5 0.657895 0.141421 0.282843 -0.552632 0.657895 0.1 0.1 -0.552632 0.657895 0.282843 0.141421 -0.552632 0.657895 0.141421 0.282843 -0.605263 0.657895 0.1 0.1 -0.605263 0.657895 0.282843 0.141421 -0.605263 0.657895 0.141421 0.282843 -0.657895 0.657895 0.1 0.1 -0.657895 0.657895 0.282843 0.141421 -0.657895 0.657895 0.141421 0.282843 -0.710526 0.657895 0.1 0.1 -0.710526 0.657895 0.282843 0.141421 -0.710526 0.657895 0.141421 0.282843 -0.763158 0.657895 0.1 0.1 -0.763158 0.657895 0.282843 0.141421 -0.763158 0.657895 0.141421 0.282843 -0.81579 0.657895 0.1 0.1 -0.81579 0.657895 0.282843 0.141421 -0.81579 0.657895 0.141421 0.282843 -0.868421 0.657895 0.1 0.1 -0.868421 0.657895 0.282843 0.141421 -0.868421 0.657895 0.141421 0.282843 -0.921053 0.657895 0.1 0.1 -0.921053 0.657895 0.282843 0.141421 -0.921053 0.657895 0.141421 0.282843 -0.973684 0.657895 0.1 0.1 -0.973684 0.657895 0.282843 0.141421 -0.973684 0.657895 0.141421 0.282843 -0.0263158 0.710526 0.1 0.1 -0.0263158 0.710526 0.282843 0.141421 -0.0263158 0.710526 0.141421 0.282843 -0.0789474 0.710526 0.1 0.1 -0.0789474 0.710526 0.282843 0.141421 -0.0789474 0.710526 0.141421 0.282843 -0.131579 0.710526 0.1 0.1 -0.131579 0.710526 0.282843 0.141421 -0.131579 0.710526 0.141421 0.282843 -0.184211 0.710526 0.1 0.1 -0.184211 0.710526 0.282843 0.141421 -0.184211 0.710526 0.141421 0.282843 -0.236842 0.710526 0.1 0.1 -0.236842 0.710526 0.282843 0.141421 -0.236842 0.710526 0.141421 0.282843 -0.289474 0.710526 0.1 0.1 -0.289474 0.710526 0.282843 0.141421 -0.289474 0.710526 0.141421 0.282843 -0.342105 0.710526 0.1 0.1 -0.342105 0.710526 0.282843 0.141421 -0.342105 0.710526 0.141421 0.282843 -0.394737 0.710526 0.1 0.1 -0.394737 0.710526 0.282843 0.141421 -0.394737 0.710526 0.141421 0.282843 -0.447368 0.710526 0.1 0.1 -0.447368 0.710526 0.282843 0.141421 -0.447368 0.710526 0.141421 0.282843 -0.5 0.710526 0.1 0.1 -0.5 0.710526 0.282843 0.141421 -0.5 0.710526 0.141421 0.282843 -0.552632 0.710526 0.1 0.1 -0.552632 0.710526 0.282843 0.141421 -0.552632 0.710526 0.141421 0.282843 -0.605263 0.710526 0.1 0.1 -0.605263 0.710526 0.282843 0.141421 -0.605263 0.710526 0.141421 0.282843 -0.657895 0.710526 0.1 0.1 -0.657895 0.710526 0.282843 0.141421 -0.657895 0.710526 0.141421 0.282843 -0.710526 0.710526 0.1 0.1 -0.710526 0.710526 0.282843 0.141421 -0.710526 0.710526 0.141421 0.282843 -0.763158 0.710526 0.1 0.1 -0.763158 0.710526 0.282843 0.141421 -0.763158 0.710526 0.141421 0.282843 -0.81579 0.710526 0.1 0.1 -0.81579 0.710526 0.282843 0.141421 -0.81579 0.710526 0.141421 0.282843 -0.868421 0.710526 0.1 0.1 -0.868421 0.710526 0.282843 0.141421 -0.868421 0.710526 0.141421 0.282843 -0.921053 0.710526 0.1 0.1 -0.921053 0.710526 0.282843 0.141421 -0.921053 0.710526 0.141421 0.282843 -0.973684 0.710526 0.1 0.1 -0.973684 0.710526 0.282843 0.141421 -0.973684 0.710526 0.141421 0.282843 -0.0263158 0.763158 0.1 0.1 -0.0263158 0.763158 0.282843 0.141421 -0.0263158 0.763158 0.141421 0.282843 -0.0789474 0.763158 0.1 0.1 -0.0789474 0.763158 0.282843 0.141421 -0.0789474 0.763158 0.141421 0.282843 -0.131579 0.763158 0.1 0.1 -0.131579 0.763158 0.282843 0.141421 -0.131579 0.763158 0.141421 0.282843 -0.184211 0.763158 0.1 0.1 -0.184211 0.763158 0.282843 0.141421 -0.184211 0.763158 0.141421 0.282843 -0.236842 0.763158 0.1 0.1 -0.236842 0.763158 0.282843 0.141421 -0.236842 0.763158 0.141421 0.282843 -0.289474 0.763158 0.1 0.1 -0.289474 0.763158 0.282843 0.141421 -0.289474 0.763158 0.141421 0.282843 -0.342105 0.763158 0.1 0.1 -0.342105 0.763158 0.282843 0.141421 -0.342105 0.763158 0.141421 0.282843 -0.394737 0.763158 0.1 0.1 -0.394737 0.763158 0.282843 0.141421 -0.394737 0.763158 0.141421 0.282843 -0.447368 0.763158 0.1 0.1 -0.447368 0.763158 0.282843 0.141421 -0.447368 0.763158 0.141421 0.282843 -0.5 0.763158 0.1 0.1 -0.5 0.763158 0.282843 0.141421 -0.5 0.763158 0.141421 0.282843 -0.552632 0.763158 0.1 0.1 -0.552632 0.763158 0.282843 0.141421 -0.552632 0.763158 0.141421 0.282843 -0.605263 0.763158 0.1 0.1 -0.605263 0.763158 0.282843 0.141421 -0.605263 0.763158 0.141421 0.282843 -0.657895 0.763158 0.1 0.1 -0.657895 0.763158 0.282843 0.141421 -0.657895 0.763158 0.141421 0.282843 -0.710526 0.763158 0.1 0.1 -0.710526 0.763158 0.282843 0.141421 -0.710526 0.763158 0.141421 0.282843 -0.763158 0.763158 0.1 0.1 -0.763158 0.763158 0.282843 0.141421 -0.763158 0.763158 0.141421 0.282843 -0.81579 0.763158 0.1 0.1 -0.81579 0.763158 0.282843 0.141421 -0.81579 0.763158 0.141421 0.282843 -0.868421 0.763158 0.1 0.1 -0.868421 0.763158 0.282843 0.141421 -0.868421 0.763158 0.141421 0.282843 -0.921053 0.763158 0.1 0.1 -0.921053 0.763158 0.282843 0.141421 -0.921053 0.763158 0.141421 0.282843 -0.973684 0.763158 0.1 0.1 -0.973684 0.763158 0.282843 0.141421 -0.973684 0.763158 0.141421 0.282843 -0.0263158 0.81579 0.1 0.1 -0.0263158 0.81579 0.282843 0.141421 -0.0263158 0.81579 0.141421 0.282843 -0.0789474 0.81579 0.1 0.1 -0.0789474 0.81579 0.282843 0.141421 -0.0789474 0.81579 0.141421 0.282843 -0.131579 0.81579 0.1 0.1 -0.131579 0.81579 0.282843 0.141421 -0.131579 0.81579 0.141421 0.282843 -0.184211 0.81579 0.1 0.1 -0.184211 0.81579 0.282843 0.141421 -0.184211 0.81579 0.141421 0.282843 -0.236842 0.81579 0.1 0.1 -0.236842 0.81579 0.282843 0.141421 -0.236842 0.81579 0.141421 0.282843 -0.289474 0.81579 0.1 0.1 -0.289474 0.81579 0.282843 0.141421 -0.289474 0.81579 0.141421 0.282843 -0.342105 0.81579 0.1 0.1 -0.342105 0.81579 0.282843 0.141421 -0.342105 0.81579 0.141421 0.282843 -0.394737 0.81579 0.1 0.1 -0.394737 0.81579 0.282843 0.141421 -0.394737 0.81579 0.141421 0.282843 -0.447368 0.81579 0.1 0.1 -0.447368 0.81579 0.282843 0.141421 -0.447368 0.81579 0.141421 0.282843 -0.5 0.81579 0.1 0.1 -0.5 0.81579 0.282843 0.141421 -0.5 0.81579 0.141421 0.282843 -0.552632 0.81579 0.1 0.1 -0.552632 0.81579 0.282843 0.141421 -0.552632 0.81579 0.141421 0.282843 -0.605263 0.81579 0.1 0.1 -0.605263 0.81579 0.282843 0.141421 -0.605263 0.81579 0.141421 0.282843 -0.657895 0.81579 0.1 0.1 -0.657895 0.81579 0.282843 0.141421 -0.657895 0.81579 0.141421 0.282843 -0.710526 0.81579 0.1 0.1 -0.710526 0.81579 0.282843 0.141421 -0.710526 0.81579 0.141421 0.282843 -0.763158 0.81579 0.1 0.1 -0.763158 0.81579 0.282843 0.141421 -0.763158 0.81579 0.141421 0.282843 -0.81579 0.81579 0.1 0.1 -0.81579 0.81579 0.282843 0.141421 -0.81579 0.81579 0.141421 0.282843 -0.868421 0.81579 0.1 0.1 -0.868421 0.81579 0.282843 0.141421 -0.868421 0.81579 0.141421 0.282843 -0.921053 0.81579 0.1 0.1 -0.921053 0.81579 0.282843 0.141421 -0.921053 0.81579 0.141421 0.282843 -0.973684 0.81579 0.1 0.1 -0.973684 0.81579 0.282843 0.141421 -0.973684 0.81579 0.141421 0.282843 -0.0263158 0.868421 0.1 0.1 -0.0263158 0.868421 0.282843 0.141421 -0.0263158 0.868421 0.141421 0.282843 -0.0789474 0.868421 0.1 0.1 -0.0789474 0.868421 0.282843 0.141421 -0.0789474 0.868421 0.141421 0.282843 -0.131579 0.868421 0.1 0.1 -0.131579 0.868421 0.282843 0.141421 -0.131579 0.868421 0.141421 0.282843 -0.184211 0.868421 0.1 0.1 -0.184211 0.868421 0.282843 0.141421 -0.184211 0.868421 0.141421 0.282843 -0.236842 0.868421 0.1 0.1 -0.236842 0.868421 0.282843 0.141421 -0.236842 0.868421 0.141421 0.282843 -0.289474 0.868421 0.1 0.1 -0.289474 0.868421 0.282843 0.141421 -0.289474 0.868421 0.141421 0.282843 -0.342105 0.868421 0.1 0.1 -0.342105 0.868421 0.282843 0.141421 -0.342105 0.868421 0.141421 0.282843 -0.394737 0.868421 0.1 0.1 -0.394737 0.868421 0.282843 0.141421 -0.394737 0.868421 0.141421 0.282843 -0.447368 0.868421 0.1 0.1 -0.447368 0.868421 0.282843 0.141421 -0.447368 0.868421 0.141421 0.282843 -0.5 0.868421 0.1 0.1 -0.5 0.868421 0.282843 0.141421 -0.5 0.868421 0.141421 0.282843 -0.552632 0.868421 0.1 0.1 -0.552632 0.868421 0.282843 0.141421 -0.552632 0.868421 0.141421 0.282843 -0.605263 0.868421 0.1 0.1 -0.605263 0.868421 0.282843 0.141421 -0.605263 0.868421 0.141421 0.282843 -0.657895 0.868421 0.1 0.1 -0.657895 0.868421 0.282843 0.141421 -0.657895 0.868421 0.141421 0.282843 -0.710526 0.868421 0.1 0.1 -0.710526 0.868421 0.282843 0.141421 -0.710526 0.868421 0.141421 0.282843 -0.763158 0.868421 0.1 0.1 -0.763158 0.868421 0.282843 0.141421 -0.763158 0.868421 0.141421 0.282843 -0.81579 0.868421 0.1 0.1 -0.81579 0.868421 0.282843 0.141421 -0.81579 0.868421 0.141421 0.282843 -0.868421 0.868421 0.1 0.1 -0.868421 0.868421 0.282843 0.141421 -0.868421 0.868421 0.141421 0.282843 -0.921053 0.868421 0.1 0.1 -0.921053 0.868421 0.282843 0.141421 -0.921053 0.868421 0.141421 0.282843 -0.973684 0.868421 0.1 0.1 -0.973684 0.868421 0.282843 0.141421 -0.973684 0.868421 0.141421 0.282843 -0.0263158 0.921053 0.1 0.1 -0.0263158 0.921053 0.282843 0.141421 -0.0263158 0.921053 0.141421 0.282843 -0.0789474 0.921053 0.1 0.1 -0.0789474 0.921053 0.282843 0.141421 -0.0789474 0.921053 0.141421 0.282843 -0.131579 0.921053 0.1 0.1 -0.131579 0.921053 0.282843 0.141421 -0.131579 0.921053 0.141421 0.282843 -0.184211 0.921053 0.1 0.1 -0.184211 0.921053 0.282843 0.141421 -0.184211 0.921053 0.141421 0.282843 -0.236842 0.921053 0.1 0.1 -0.236842 0.921053 0.282843 0.141421 -0.236842 0.921053 0.141421 0.282843 -0.289474 0.921053 0.1 0.1 -0.289474 0.921053 0.282843 0.141421 -0.289474 0.921053 0.141421 0.282843 -0.342105 0.921053 0.1 0.1 -0.342105 0.921053 0.282843 0.141421 -0.342105 0.921053 0.141421 0.282843 -0.394737 0.921053 0.1 0.1 -0.394737 0.921053 0.282843 0.141421 -0.394737 0.921053 0.141421 0.282843 -0.447368 0.921053 0.1 0.1 -0.447368 0.921053 0.282843 0.141421 -0.447368 0.921053 0.141421 0.282843 -0.5 0.921053 0.1 0.1 -0.5 0.921053 0.282843 0.141421 -0.5 0.921053 0.141421 0.282843 -0.552632 0.921053 0.1 0.1 -0.552632 0.921053 0.282843 0.141421 -0.552632 0.921053 0.141421 0.282843 -0.605263 0.921053 0.1 0.1 -0.605263 0.921053 0.282843 0.141421 -0.605263 0.921053 0.141421 0.282843 -0.657895 0.921053 0.1 0.1 -0.657895 0.921053 0.282843 0.141421 -0.657895 0.921053 0.141421 0.282843 -0.710526 0.921053 0.1 0.1 -0.710526 0.921053 0.282843 0.141421 -0.710526 0.921053 0.141421 0.282843 -0.763158 0.921053 0.1 0.1 -0.763158 0.921053 0.282843 0.141421 -0.763158 0.921053 0.141421 0.282843 -0.81579 0.921053 0.1 0.1 -0.81579 0.921053 0.282843 0.141421 -0.81579 0.921053 0.141421 0.282843 -0.868421 0.921053 0.1 0.1 -0.868421 0.921053 0.282843 0.141421 -0.868421 0.921053 0.141421 0.282843 -0.921053 0.921053 0.1 0.1 -0.921053 0.921053 0.282843 0.141421 -0.921053 0.921053 0.141421 0.282843 -0.973684 0.921053 0.1 0.1 -0.973684 0.921053 0.282843 0.141421 -0.973684 0.921053 0.141421 0.282843 -0.0263158 0.973684 0.1 0.1 -0.0263158 0.973684 0.282843 0.141421 -0.0263158 0.973684 0.141421 0.282843 -0.0789474 0.973684 0.1 0.1 -0.0789474 0.973684 0.282843 0.141421 -0.0789474 0.973684 0.141421 0.282843 -0.131579 0.973684 0.1 0.1 -0.131579 0.973684 0.282843 0.141421 -0.131579 0.973684 0.141421 0.282843 -0.184211 0.973684 0.1 0.1 -0.184211 0.973684 0.282843 0.141421 -0.184211 0.973684 0.141421 0.282843 -0.236842 0.973684 0.1 0.1 -0.236842 0.973684 0.282843 0.141421 -0.236842 0.973684 0.141421 0.282843 -0.289474 0.973684 0.1 0.1 -0.289474 0.973684 0.282843 0.141421 -0.289474 0.973684 0.141421 0.282843 -0.342105 0.973684 0.1 0.1 -0.342105 0.973684 0.282843 0.141421 -0.342105 0.973684 0.141421 0.282843 -0.394737 0.973684 0.1 0.1 -0.394737 0.973684 0.282843 0.141421 -0.394737 0.973684 0.141421 0.282843 -0.447368 0.973684 0.1 0.1 -0.447368 0.973684 0.282843 0.141421 -0.447368 0.973684 0.141421 0.282843 -0.5 0.973684 0.1 0.1 -0.5 0.973684 0.282843 0.141421 -0.5 0.973684 0.141421 0.282843 -0.552632 0.973684 0.1 0.1 -0.552632 0.973684 0.282843 0.141421 -0.552632 0.973684 0.141421 0.282843 -0.605263 0.973684 0.1 0.1 -0.605263 0.973684 0.282843 0.141421 -0.605263 0.973684 0.141421 0.282843 -0.657895 0.973684 0.1 0.1 -0.657895 0.973684 0.282843 0.141421 -0.657895 0.973684 0.141421 0.282843 -0.710526 0.973684 0.1 0.1 -0.710526 0.973684 0.282843 0.141421 -0.710526 0.973684 0.141421 0.282843 -0.763158 0.973684 0.1 0.1 -0.763158 0.973684 0.282843 0.141421 -0.763158 0.973684 0.141421 0.282843 -0.81579 0.973684 0.1 0.1 -0.81579 0.973684 0.282843 0.141421 -0.81579 0.973684 0.141421 0.282843 -0.868421 0.973684 0.1 0.1 -0.868421 0.973684 0.282843 0.141421 -0.868421 0.973684 0.141421 0.282843 -0.921053 0.973684 0.1 0.1 -0.921053 0.973684 0.282843 0.141421 -0.921053 0.973684 0.141421 0.282843 -0.973684 0.973684 0.1 0.1 -0.973684 0.973684 0.282843 0.141421 -0.973684 0.973684 0.141421 0.282843 -0.05 0.05 0.35 0.35 -0.05 0.05 0.494975 0.247487 -0.05 0.05 0.247487 0.494975 -0.05 0.05 0.606218 0.202073 -0.05 0.05 0.202062 0.606248 -0.05 0.05 0.41833 0.41833 -0.15 0.05 0.35 0.35 -0.15 0.05 0.494975 0.247487 -0.15 0.05 0.247487 0.494975 -0.15 0.05 0.606218 0.202073 -0.15 0.05 0.202062 0.606248 -0.15 0.05 0.41833 0.41833 -0.25 0.05 0.35 0.35 -0.25 0.05 0.494975 0.247487 -0.25 0.05 0.247487 0.494975 -0.25 0.05 0.606218 0.202073 -0.25 0.05 0.202062 0.606248 -0.25 0.05 0.41833 0.41833 -0.35 0.05 0.35 0.35 -0.35 0.05 0.494975 0.247487 -0.35 0.05 0.247487 0.494975 -0.35 0.05 0.606218 0.202073 -0.35 0.05 0.202062 0.606248 -0.35 0.05 0.41833 0.41833 -0.45 0.05 0.35 0.35 -0.45 0.05 0.494975 0.247487 -0.45 0.05 0.247487 0.494975 -0.45 0.05 0.606218 0.202073 -0.45 0.05 0.202062 0.606248 -0.45 0.05 0.41833 0.41833 -0.55 0.05 0.35 0.35 -0.55 0.05 0.494975 0.247487 -0.55 0.05 0.247487 0.494975 -0.55 0.05 0.606218 0.202073 -0.55 0.05 0.202062 0.606248 -0.55 0.05 0.41833 0.41833 -0.65 0.05 0.35 0.35 -0.65 0.05 0.494975 0.247487 -0.65 0.05 0.247487 0.494975 -0.65 0.05 0.606218 0.202073 -0.65 0.05 0.202062 0.606248 -0.65 0.05 0.41833 0.41833 -0.75 0.05 0.35 0.35 -0.75 0.05 0.494975 0.247487 -0.75 0.05 0.247487 0.494975 -0.75 0.05 0.606218 0.202073 -0.75 0.05 0.202062 0.606248 -0.75 0.05 0.41833 0.41833 -0.85 0.05 0.35 0.35 -0.85 0.05 0.494975 0.247487 -0.85 0.05 0.247487 0.494975 -0.85 0.05 0.606218 0.202073 -0.85 0.05 0.202062 0.606248 -0.85 0.05 0.41833 0.41833 -0.95 0.05 0.35 0.35 -0.95 0.05 0.494975 0.247487 -0.95 0.05 0.247487 0.494975 -0.95 0.05 0.606218 0.202073 -0.95 0.05 0.202063 0.606248 -0.95 0.05 0.41833 0.41833 -0.05 0.15 0.35 0.35 -0.05 0.15 0.494975 0.247487 -0.05 0.15 0.247487 0.494975 -0.05 0.15 0.606218 0.202073 -0.05 0.15 0.202062 0.606248 -0.05 0.15 0.41833 0.41833 -0.15 0.15 0.35 0.35 -0.15 0.15 0.494975 0.247487 -0.15 0.15 0.247487 0.494975 -0.15 0.15 0.606218 0.202073 -0.15 0.15 0.202062 0.606248 -0.15 0.15 0.41833 0.41833 -0.25 0.15 0.35 0.35 -0.25 0.15 0.494975 0.247487 -0.25 0.15 0.247487 0.494975 -0.25 0.15 0.606218 0.202073 -0.25 0.15 0.202062 0.606248 -0.25 0.15 0.41833 0.41833 -0.35 0.15 0.35 0.35 -0.35 0.15 0.494975 0.247487 -0.35 0.15 0.247487 0.494975 -0.35 0.15 0.606218 0.202073 -0.35 0.15 0.202062 0.606248 -0.35 0.15 0.41833 0.41833 -0.45 0.15 0.35 0.35 -0.45 0.15 0.494975 0.247487 -0.45 0.15 0.247487 0.494975 -0.45 0.15 0.606218 0.202073 -0.45 0.15 0.202062 0.606248 -0.45 0.15 0.41833 0.41833 -0.55 0.15 0.35 0.35 -0.55 0.15 0.494975 0.247487 -0.55 0.15 0.247487 0.494975 -0.55 0.15 0.606218 0.202073 -0.55 0.15 0.202062 0.606248 -0.55 0.15 0.41833 0.41833 -0.65 0.15 0.35 0.35 -0.65 0.15 0.494975 0.247487 -0.65 0.15 0.247487 0.494975 -0.65 0.15 0.606218 0.202073 -0.65 0.15 0.202062 0.606248 -0.65 0.15 0.41833 0.41833 -0.75 0.15 0.35 0.35 -0.75 0.15 0.494975 0.247487 -0.75 0.15 0.247487 0.494975 -0.75 0.15 0.606218 0.202073 -0.75 0.15 0.202062 0.606248 -0.75 0.15 0.41833 0.41833 -0.85 0.15 0.35 0.35 -0.85 0.15 0.494975 0.247487 -0.85 0.15 0.247487 0.494975 -0.85 0.15 0.606218 0.202073 -0.85 0.15 0.202062 0.606248 -0.85 0.15 0.41833 0.41833 -0.95 0.15 0.35 0.35 -0.95 0.15 0.494975 0.247487 -0.95 0.15 0.247487 0.494975 -0.95 0.15 0.606218 0.202073 -0.95 0.15 0.202063 0.606248 -0.95 0.15 0.41833 0.41833 -0.05 0.25 0.35 0.35 -0.05 0.25 0.494975 0.247487 -0.05 0.25 0.247487 0.494975 -0.05 0.25 0.606218 0.202073 -0.05 0.25 0.202062 0.606248 -0.05 0.25 0.41833 0.41833 -0.15 0.25 0.35 0.35 -0.15 0.25 0.494975 0.247487 -0.15 0.25 0.247487 0.494975 -0.15 0.25 0.606218 0.202073 -0.15 0.25 0.202062 0.606248 -0.15 0.25 0.41833 0.41833 -0.25 0.25 0.35 0.35 -0.25 0.25 0.494975 0.247487 -0.25 0.25 0.247487 0.494975 -0.25 0.25 0.606218 0.202073 -0.25 0.25 0.202062 0.606248 -0.25 0.25 0.41833 0.41833 -0.35 0.25 0.35 0.35 -0.35 0.25 0.494975 0.247487 -0.35 0.25 0.247487 0.494975 -0.35 0.25 0.606218 0.202073 -0.35 0.25 0.202062 0.606248 -0.35 0.25 0.41833 0.41833 -0.45 0.25 0.35 0.35 -0.45 0.25 0.494975 0.247487 -0.45 0.25 0.247487 0.494975 -0.45 0.25 0.606218 0.202073 -0.45 0.25 0.202062 0.606248 -0.45 0.25 0.41833 0.41833 -0.55 0.25 0.35 0.35 -0.55 0.25 0.494975 0.247487 -0.55 0.25 0.247487 0.494975 -0.55 0.25 0.606218 0.202073 -0.55 0.25 0.202062 0.606248 -0.55 0.25 0.41833 0.41833 -0.65 0.25 0.35 0.35 -0.65 0.25 0.494975 0.247487 -0.65 0.25 0.247487 0.494975 -0.65 0.25 0.606218 0.202073 -0.65 0.25 0.202062 0.606248 -0.65 0.25 0.41833 0.41833 -0.75 0.25 0.35 0.35 -0.75 0.25 0.494975 0.247487 -0.75 0.25 0.247487 0.494975 -0.75 0.25 0.606218 0.202073 -0.75 0.25 0.202062 0.606248 -0.75 0.25 0.41833 0.41833 -0.85 0.25 0.35 0.35 -0.85 0.25 0.494975 0.247487 -0.85 0.25 0.247487 0.494975 -0.85 0.25 0.606218 0.202073 -0.85 0.25 0.202062 0.606248 -0.85 0.25 0.41833 0.41833 -0.95 0.25 0.35 0.35 -0.95 0.25 0.494975 0.247487 -0.95 0.25 0.247487 0.494975 -0.95 0.25 0.606218 0.202073 -0.95 0.25 0.202063 0.606248 -0.95 0.25 0.41833 0.41833 -0.05 0.35 0.35 0.35 -0.05 0.35 0.494975 0.247487 -0.05 0.35 0.247487 0.494975 -0.05 0.35 0.606218 0.202073 -0.05 0.35 0.202062 0.606248 -0.05 0.35 0.41833 0.41833 -0.15 0.35 0.35 0.35 -0.15 0.35 0.494975 0.247487 -0.15 0.35 0.247487 0.494975 -0.15 0.35 0.606218 0.202073 -0.15 0.35 0.202062 0.606248 -0.15 0.35 0.41833 0.41833 -0.25 0.35 0.35 0.35 -0.25 0.35 0.494975 0.247487 -0.25 0.35 0.247487 0.494975 -0.25 0.35 0.606218 0.202073 -0.25 0.35 0.202062 0.606248 -0.25 0.35 0.41833 0.41833 -0.35 0.35 0.35 0.35 -0.35 0.35 0.494975 0.247487 -0.35 0.35 0.247487 0.494975 -0.35 0.35 0.606218 0.202073 -0.35 0.35 0.202062 0.606248 -0.35 0.35 0.41833 0.41833 -0.45 0.35 0.35 0.35 -0.45 0.35 0.494975 0.247487 -0.45 0.35 0.247487 0.494975 -0.45 0.35 0.606218 0.202073 -0.45 0.35 0.202062 0.606248 -0.45 0.35 0.41833 0.41833 -0.55 0.35 0.35 0.35 -0.55 0.35 0.494975 0.247487 -0.55 0.35 0.247487 0.494975 -0.55 0.35 0.606218 0.202073 -0.55 0.35 0.202062 0.606248 -0.55 0.35 0.41833 0.41833 -0.65 0.35 0.35 0.35 -0.65 0.35 0.494975 0.247487 -0.65 0.35 0.247487 0.494975 -0.65 0.35 0.606218 0.202073 -0.65 0.35 0.202062 0.606248 -0.65 0.35 0.41833 0.41833 -0.75 0.35 0.35 0.35 -0.75 0.35 0.494975 0.247487 -0.75 0.35 0.247487 0.494975 -0.75 0.35 0.606218 0.202073 -0.75 0.35 0.202062 0.606248 -0.75 0.35 0.41833 0.41833 -0.85 0.35 0.35 0.35 -0.85 0.35 0.494975 0.247487 -0.85 0.35 0.247487 0.494975 -0.85 0.35 0.606218 0.202073 -0.85 0.35 0.202062 0.606248 -0.85 0.35 0.41833 0.41833 -0.95 0.35 0.35 0.35 -0.95 0.35 0.494975 0.247487 -0.95 0.35 0.247487 0.494975 -0.95 0.35 0.606218 0.202073 -0.95 0.35 0.202063 0.606248 -0.95 0.35 0.41833 0.41833 -0.05 0.45 0.35 0.35 -0.05 0.45 0.494975 0.247487 -0.05 0.45 0.247487 0.494975 -0.05 0.45 0.606218 0.202073 -0.05 0.45 0.202062 0.606248 -0.05 0.45 0.41833 0.41833 -0.15 0.45 0.35 0.35 -0.15 0.45 0.494975 0.247487 -0.15 0.45 0.247487 0.494975 -0.15 0.45 0.606218 0.202073 -0.15 0.45 0.202062 0.606248 -0.15 0.45 0.41833 0.41833 -0.25 0.45 0.35 0.35 -0.25 0.45 0.494975 0.247487 -0.25 0.45 0.247487 0.494975 -0.25 0.45 0.606218 0.202073 -0.25 0.45 0.202062 0.606248 -0.25 0.45 0.41833 0.41833 -0.35 0.45 0.35 0.35 -0.35 0.45 0.494975 0.247487 -0.35 0.45 0.247487 0.494975 -0.35 0.45 0.606218 0.202073 -0.35 0.45 0.202062 0.606248 -0.35 0.45 0.41833 0.41833 -0.45 0.45 0.35 0.35 -0.45 0.45 0.494975 0.247487 -0.45 0.45 0.247487 0.494975 -0.45 0.45 0.606218 0.202073 -0.45 0.45 0.202062 0.606248 -0.45 0.45 0.41833 0.41833 -0.55 0.45 0.35 0.35 -0.55 0.45 0.494975 0.247487 -0.55 0.45 0.247487 0.494975 -0.55 0.45 0.606218 0.202073 -0.55 0.45 0.202062 0.606248 -0.55 0.45 0.41833 0.41833 -0.65 0.45 0.35 0.35 -0.65 0.45 0.494975 0.247487 -0.65 0.45 0.247487 0.494975 -0.65 0.45 0.606218 0.202073 -0.65 0.45 0.202062 0.606248 -0.65 0.45 0.41833 0.41833 -0.75 0.45 0.35 0.35 -0.75 0.45 0.494975 0.247487 -0.75 0.45 0.247487 0.494975 -0.75 0.45 0.606218 0.202073 -0.75 0.45 0.202062 0.606248 -0.75 0.45 0.41833 0.41833 -0.85 0.45 0.35 0.35 -0.85 0.45 0.494975 0.247487 -0.85 0.45 0.247487 0.494975 -0.85 0.45 0.606218 0.202073 -0.85 0.45 0.202062 0.606248 -0.85 0.45 0.41833 0.41833 -0.95 0.45 0.35 0.35 -0.95 0.45 0.494975 0.247487 -0.95 0.45 0.247487 0.494975 -0.95 0.45 0.606218 0.202073 -0.95 0.45 0.202063 0.606248 -0.95 0.45 0.41833 0.41833 -0.05 0.55 0.35 0.35 -0.05 0.55 0.494975 0.247487 -0.05 0.55 0.247487 0.494975 -0.05 0.55 0.606218 0.202073 -0.05 0.55 0.202062 0.606248 -0.05 0.55 0.41833 0.41833 -0.15 0.55 0.35 0.35 -0.15 0.55 0.494975 0.247487 -0.15 0.55 0.247487 0.494975 -0.15 0.55 0.606218 0.202073 -0.15 0.55 0.202062 0.606248 -0.15 0.55 0.41833 0.41833 -0.25 0.55 0.35 0.35 -0.25 0.55 0.494975 0.247487 -0.25 0.55 0.247487 0.494975 -0.25 0.55 0.606218 0.202073 -0.25 0.55 0.202062 0.606248 -0.25 0.55 0.41833 0.41833 -0.35 0.55 0.35 0.35 -0.35 0.55 0.494975 0.247487 -0.35 0.55 0.247487 0.494975 -0.35 0.55 0.606218 0.202073 -0.35 0.55 0.202062 0.606248 -0.35 0.55 0.41833 0.41833 -0.45 0.55 0.35 0.35 -0.45 0.55 0.494975 0.247487 -0.45 0.55 0.247487 0.494975 -0.45 0.55 0.606218 0.202073 -0.45 0.55 0.202062 0.606248 -0.45 0.55 0.41833 0.41833 -0.55 0.55 0.35 0.35 -0.55 0.55 0.494975 0.247487 -0.55 0.55 0.247487 0.494975 -0.55 0.55 0.606218 0.202073 -0.55 0.55 0.202062 0.606248 -0.55 0.55 0.41833 0.41833 -0.65 0.55 0.35 0.35 -0.65 0.55 0.494975 0.247487 -0.65 0.55 0.247487 0.494975 -0.65 0.55 0.606218 0.202073 -0.65 0.55 0.202062 0.606248 -0.65 0.55 0.41833 0.41833 -0.75 0.55 0.35 0.35 -0.75 0.55 0.494975 0.247487 -0.75 0.55 0.247487 0.494975 -0.75 0.55 0.606218 0.202073 -0.75 0.55 0.202062 0.606248 -0.75 0.55 0.41833 0.41833 -0.85 0.55 0.35 0.35 -0.85 0.55 0.494975 0.247487 -0.85 0.55 0.247487 0.494975 -0.85 0.55 0.606218 0.202073 -0.85 0.55 0.202062 0.606248 -0.85 0.55 0.41833 0.41833 -0.95 0.55 0.35 0.35 -0.95 0.55 0.494975 0.247487 -0.95 0.55 0.247487 0.494975 -0.95 0.55 0.606218 0.202073 -0.95 0.55 0.202063 0.606248 -0.95 0.55 0.41833 0.41833 -0.05 0.65 0.35 0.35 -0.05 0.65 0.494975 0.247487 -0.05 0.65 0.247487 0.494975 -0.05 0.65 0.606218 0.202073 -0.05 0.65 0.202062 0.606248 -0.05 0.65 0.41833 0.41833 -0.15 0.65 0.35 0.35 -0.15 0.65 0.494975 0.247487 -0.15 0.65 0.247487 0.494975 -0.15 0.65 0.606218 0.202073 -0.15 0.65 0.202062 0.606248 -0.15 0.65 0.41833 0.41833 -0.25 0.65 0.35 0.35 -0.25 0.65 0.494975 0.247487 -0.25 0.65 0.247487 0.494975 -0.25 0.65 0.606218 0.202073 -0.25 0.65 0.202062 0.606248 -0.25 0.65 0.41833 0.41833 -0.35 0.65 0.35 0.35 -0.35 0.65 0.494975 0.247487 -0.35 0.65 0.247487 0.494975 -0.35 0.65 0.606218 0.202073 -0.35 0.65 0.202062 0.606248 -0.35 0.65 0.41833 0.41833 -0.45 0.65 0.35 0.35 -0.45 0.65 0.494975 0.247487 -0.45 0.65 0.247487 0.494975 -0.45 0.65 0.606218 0.202073 -0.45 0.65 0.202062 0.606248 -0.45 0.65 0.41833 0.41833 -0.55 0.65 0.35 0.35 -0.55 0.65 0.494975 0.247487 -0.55 0.65 0.247487 0.494975 -0.55 0.65 0.606218 0.202073 -0.55 0.65 0.202062 0.606248 -0.55 0.65 0.41833 0.41833 -0.65 0.65 0.35 0.35 -0.65 0.65 0.494975 0.247487 -0.65 0.65 0.247487 0.494975 -0.65 0.65 0.606218 0.202073 -0.65 0.65 0.202062 0.606248 -0.65 0.65 0.41833 0.41833 -0.75 0.65 0.35 0.35 -0.75 0.65 0.494975 0.247487 -0.75 0.65 0.247487 0.494975 -0.75 0.65 0.606218 0.202073 -0.75 0.65 0.202062 0.606248 -0.75 0.65 0.41833 0.41833 -0.85 0.65 0.35 0.35 -0.85 0.65 0.494975 0.247487 -0.85 0.65 0.247487 0.494975 -0.85 0.65 0.606218 0.202073 -0.85 0.65 0.202062 0.606248 -0.85 0.65 0.41833 0.41833 -0.95 0.65 0.35 0.35 -0.95 0.65 0.494975 0.247487 -0.95 0.65 0.247487 0.494975 -0.95 0.65 0.606218 0.202073 -0.95 0.65 0.202063 0.606248 -0.95 0.65 0.41833 0.41833 -0.05 0.75 0.35 0.35 -0.05 0.75 0.494975 0.247487 -0.05 0.75 0.247487 0.494975 -0.05 0.75 0.606218 0.202073 -0.05 0.75 0.202062 0.606248 -0.05 0.75 0.41833 0.41833 -0.15 0.75 0.35 0.35 -0.15 0.75 0.494975 0.247487 -0.15 0.75 0.247487 0.494975 -0.15 0.75 0.606218 0.202073 -0.15 0.75 0.202062 0.606248 -0.15 0.75 0.41833 0.41833 -0.25 0.75 0.35 0.35 -0.25 0.75 0.494975 0.247487 -0.25 0.75 0.247487 0.494975 -0.25 0.75 0.606218 0.202073 -0.25 0.75 0.202062 0.606248 -0.25 0.75 0.41833 0.41833 -0.35 0.75 0.35 0.35 -0.35 0.75 0.494975 0.247487 -0.35 0.75 0.247487 0.494975 -0.35 0.75 0.606218 0.202073 -0.35 0.75 0.202062 0.606248 -0.35 0.75 0.41833 0.41833 -0.45 0.75 0.35 0.35 -0.45 0.75 0.494975 0.247487 -0.45 0.75 0.247487 0.494975 -0.45 0.75 0.606218 0.202073 -0.45 0.75 0.202062 0.606248 -0.45 0.75 0.41833 0.41833 -0.55 0.75 0.35 0.35 -0.55 0.75 0.494975 0.247487 -0.55 0.75 0.247487 0.494975 -0.55 0.75 0.606218 0.202073 -0.55 0.75 0.202062 0.606248 -0.55 0.75 0.41833 0.41833 -0.65 0.75 0.35 0.35 -0.65 0.75 0.494975 0.247487 -0.65 0.75 0.247487 0.494975 -0.65 0.75 0.606218 0.202073 -0.65 0.75 0.202062 0.606248 -0.65 0.75 0.41833 0.41833 -0.75 0.75 0.35 0.35 -0.75 0.75 0.494975 0.247487 -0.75 0.75 0.247487 0.494975 -0.75 0.75 0.606218 0.202073 -0.75 0.75 0.202062 0.606248 -0.75 0.75 0.41833 0.41833 -0.85 0.75 0.35 0.35 -0.85 0.75 0.494975 0.247487 -0.85 0.75 0.247487 0.494975 -0.85 0.75 0.606218 0.202073 -0.85 0.75 0.202062 0.606248 -0.85 0.75 0.41833 0.41833 -0.95 0.75 0.35 0.35 -0.95 0.75 0.494975 0.247487 -0.95 0.75 0.247487 0.494975 -0.95 0.75 0.606218 0.202073 -0.95 0.75 0.202063 0.606248 -0.95 0.75 0.41833 0.41833 -0.05 0.85 0.35 0.35 -0.05 0.85 0.494975 0.247487 -0.05 0.85 0.247487 0.494975 -0.05 0.85 0.606218 0.202073 -0.05 0.85 0.202062 0.606248 -0.05 0.85 0.41833 0.41833 -0.15 0.85 0.35 0.35 -0.15 0.85 0.494975 0.247487 -0.15 0.85 0.247487 0.494975 -0.15 0.85 0.606218 0.202073 -0.15 0.85 0.202062 0.606248 -0.15 0.85 0.41833 0.41833 -0.25 0.85 0.35 0.35 -0.25 0.85 0.494975 0.247487 -0.25 0.85 0.247487 0.494975 -0.25 0.85 0.606218 0.202073 -0.25 0.85 0.202062 0.606248 -0.25 0.85 0.41833 0.41833 -0.35 0.85 0.35 0.35 -0.35 0.85 0.494975 0.247487 -0.35 0.85 0.247487 0.494975 -0.35 0.85 0.606218 0.202073 -0.35 0.85 0.202062 0.606248 -0.35 0.85 0.41833 0.41833 -0.45 0.85 0.35 0.35 -0.45 0.85 0.494975 0.247487 -0.45 0.85 0.247487 0.494975 -0.45 0.85 0.606218 0.202073 -0.45 0.85 0.202062 0.606248 -0.45 0.85 0.41833 0.41833 -0.55 0.85 0.35 0.35 -0.55 0.85 0.494975 0.247487 -0.55 0.85 0.247487 0.494975 -0.55 0.85 0.606218 0.202073 -0.55 0.85 0.202062 0.606248 -0.55 0.85 0.41833 0.41833 -0.65 0.85 0.35 0.35 -0.65 0.85 0.494975 0.247487 -0.65 0.85 0.247487 0.494975 -0.65 0.85 0.606218 0.202073 -0.65 0.85 0.202062 0.606248 -0.65 0.85 0.41833 0.41833 -0.75 0.85 0.35 0.35 -0.75 0.85 0.494975 0.247487 -0.75 0.85 0.247487 0.494975 -0.75 0.85 0.606218 0.202073 -0.75 0.85 0.202062 0.606248 -0.75 0.85 0.41833 0.41833 -0.85 0.85 0.35 0.35 -0.85 0.85 0.494975 0.247487 -0.85 0.85 0.247487 0.494975 -0.85 0.85 0.606218 0.202073 -0.85 0.85 0.202062 0.606248 -0.85 0.85 0.41833 0.41833 -0.95 0.85 0.35 0.35 -0.95 0.85 0.494975 0.247487 -0.95 0.85 0.247487 0.494975 -0.95 0.85 0.606218 0.202073 -0.95 0.85 0.202063 0.606248 -0.95 0.85 0.41833 0.41833 -0.05 0.95 0.35 0.35 -0.05 0.95 0.494975 0.247487 -0.05 0.95 0.247487 0.494975 -0.05 0.95 0.606218 0.202073 -0.05 0.95 0.202062 0.606248 -0.05 0.95 0.41833 0.41833 -0.15 0.95 0.35 0.35 -0.15 0.95 0.494975 0.247487 -0.15 0.95 0.247487 0.494975 -0.15 0.95 0.606218 0.202073 -0.15 0.95 0.202062 0.606248 -0.15 0.95 0.41833 0.41833 -0.25 0.95 0.35 0.35 -0.25 0.95 0.494975 0.247487 -0.25 0.95 0.247487 0.494975 -0.25 0.95 0.606218 0.202073 -0.25 0.95 0.202062 0.606248 -0.25 0.95 0.41833 0.41833 -0.35 0.95 0.35 0.35 -0.35 0.95 0.494975 0.247487 -0.35 0.95 0.247487 0.494975 -0.35 0.95 0.606218 0.202073 -0.35 0.95 0.202062 0.606248 -0.35 0.95 0.41833 0.41833 -0.45 0.95 0.35 0.35 -0.45 0.95 0.494975 0.247487 -0.45 0.95 0.247487 0.494975 -0.45 0.95 0.606218 0.202073 -0.45 0.95 0.202062 0.606248 -0.45 0.95 0.41833 0.41833 -0.55 0.95 0.35 0.35 -0.55 0.95 0.494975 0.247487 -0.55 0.95 0.247487 0.494975 -0.55 0.95 0.606218 0.202073 -0.55 0.95 0.202062 0.606248 -0.55 0.95 0.41833 0.41833 -0.65 0.95 0.35 0.35 -0.65 0.95 0.494975 0.247487 -0.65 0.95 0.247487 0.494975 -0.65 0.95 0.606218 0.202073 -0.65 0.95 0.202062 0.606248 -0.65 0.95 0.41833 0.41833 -0.75 0.95 0.35 0.35 -0.75 0.95 0.494975 0.247487 -0.75 0.95 0.247487 0.494975 -0.75 0.95 0.606218 0.202073 -0.75 0.95 0.202062 0.606248 -0.75 0.95 0.41833 0.41833 -0.85 0.95 0.35 0.35 -0.85 0.95 0.494975 0.247487 -0.85 0.95 0.247487 0.494975 -0.85 0.95 0.606218 0.202073 -0.85 0.95 0.202062 0.606248 -0.85 0.95 0.41833 0.41833 -0.95 0.95 0.35 0.35 -0.95 0.95 0.494975 0.247487 -0.95 0.95 0.247487 0.494975 -0.95 0.95 0.606218 0.202073 -0.95 0.95 0.202063 0.606248 -0.95 0.95 0.41833 0.41833 -0.1 0.1 0.5 0.5 -0.1 0.1 0.707107 0.353553 -0.1 0.1 0.353553 0.707107 -0.1 0.1 0.866025 0.288675 -0.1 0.1 0.288661 0.866069 -0.1 0.1 0.570088 0.570088 -0.3 0.1 0.5 0.5 -0.3 0.1 0.707107 0.353553 -0.3 0.1 0.353553 0.707107 -0.3 0.1 0.866025 0.288675 -0.3 0.1 0.288661 0.866069 -0.3 0.1 0.570088 0.570088 -0.5 0.1 0.5 0.5 -0.5 0.1 0.707107 0.353553 -0.5 0.1 0.353553 0.707107 -0.5 0.1 0.866025 0.288675 -0.5 0.1 0.288661 0.866069 -0.5 0.1 0.570088 0.570088 -0.7 0.1 0.5 0.5 -0.7 0.1 0.707107 0.353553 -0.7 0.1 0.353553 0.707107 -0.7 0.1 0.866025 0.288675 -0.7 0.1 0.288661 0.866069 -0.7 0.1 0.570088 0.570088 -0.9 0.1 0.5 0.5 -0.9 0.1 0.707107 0.353553 -0.9 0.1 0.353553 0.707107 -0.9 0.1 0.866025 0.288675 -0.9 0.1 0.288661 0.866069 -0.9 0.1 0.570088 0.570088 -0.1 0.3 0.5 0.5 -0.1 0.3 0.707107 0.353553 -0.1 0.3 0.353553 0.707107 -0.1 0.3 0.866025 0.288675 -0.1 0.3 0.288661 0.866069 -0.1 0.3 0.570088 0.570088 -0.3 0.3 0.5 0.5 -0.3 0.3 0.707107 0.353553 -0.3 0.3 0.353553 0.707107 -0.3 0.3 0.866025 0.288675 -0.3 0.3 0.288661 0.866069 -0.3 0.3 0.570088 0.570088 -0.5 0.3 0.5 0.5 -0.5 0.3 0.707107 0.353553 -0.5 0.3 0.353553 0.707107 -0.5 0.3 0.866025 0.288675 -0.5 0.3 0.288661 0.866069 -0.5 0.3 0.570088 0.570088 -0.7 0.3 0.5 0.5 -0.7 0.3 0.707107 0.353553 -0.7 0.3 0.353553 0.707107 -0.7 0.3 0.866025 0.288675 -0.7 0.3 0.288661 0.866069 -0.7 0.3 0.570088 0.570088 -0.9 0.3 0.5 0.5 -0.9 0.3 0.707107 0.353553 -0.9 0.3 0.353553 0.707107 -0.9 0.3 0.866025 0.288675 -0.9 0.3 0.288661 0.866069 -0.9 0.3 0.570088 0.570088 -0.1 0.5 0.5 0.5 -0.1 0.5 0.707107 0.353553 -0.1 0.5 0.353553 0.707107 -0.1 0.5 0.866025 0.288675 -0.1 0.5 0.288661 0.866069 -0.1 0.5 0.570088 0.570088 -0.3 0.5 0.5 0.5 -0.3 0.5 0.707107 0.353553 -0.3 0.5 0.353553 0.707107 -0.3 0.5 0.866025 0.288675 -0.3 0.5 0.288661 0.866069 -0.3 0.5 0.570088 0.570088 -0.5 0.5 0.5 0.5 -0.5 0.5 0.707107 0.353553 -0.5 0.5 0.353553 0.707107 -0.5 0.5 0.866025 0.288675 -0.5 0.5 0.288661 0.866069 -0.5 0.5 0.570088 0.570088 -0.7 0.5 0.5 0.5 -0.7 0.5 0.707107 0.353553 -0.7 0.5 0.353553 0.707107 -0.7 0.5 0.866025 0.288675 -0.7 0.5 0.288661 0.866069 -0.7 0.5 0.570088 0.570088 -0.9 0.5 0.5 0.5 -0.9 0.5 0.707107 0.353553 -0.9 0.5 0.353553 0.707107 -0.9 0.5 0.866025 0.288675 -0.9 0.5 0.288661 0.866069 -0.9 0.5 0.570088 0.570088 -0.1 0.7 0.5 0.5 -0.1 0.7 0.707107 0.353553 -0.1 0.7 0.353553 0.707107 -0.1 0.7 0.866025 0.288675 -0.1 0.7 0.288661 0.866069 -0.1 0.7 0.570088 0.570088 -0.3 0.7 0.5 0.5 -0.3 0.7 0.707107 0.353553 -0.3 0.7 0.353553 0.707107 -0.3 0.7 0.866025 0.288675 -0.3 0.7 0.288661 0.866069 -0.3 0.7 0.570088 0.570088 -0.5 0.7 0.5 0.5 -0.5 0.7 0.707107 0.353553 -0.5 0.7 0.353553 0.707107 -0.5 0.7 0.866025 0.288675 -0.5 0.7 0.288661 0.866069 -0.5 0.7 0.570088 0.570088 -0.7 0.7 0.5 0.5 -0.7 0.7 0.707107 0.353553 -0.7 0.7 0.353553 0.707107 -0.7 0.7 0.866025 0.288675 -0.7 0.7 0.288661 0.866069 -0.7 0.7 0.570088 0.570088 -0.9 0.7 0.5 0.5 -0.9 0.7 0.707107 0.353553 -0.9 0.7 0.353553 0.707107 -0.9 0.7 0.866025 0.288675 -0.9 0.7 0.288661 0.866069 -0.9 0.7 0.570088 0.570088 -0.1 0.9 0.5 0.5 -0.1 0.9 0.707107 0.353553 -0.1 0.9 0.353553 0.707107 -0.1 0.9 0.866025 0.288675 -0.1 0.9 0.288661 0.866069 -0.1 0.9 0.570088 0.570088 -0.3 0.9 0.5 0.5 -0.3 0.9 0.707107 0.353553 -0.3 0.9 0.353553 0.707107 -0.3 0.9 0.866025 0.288675 -0.3 0.9 0.288661 0.866069 -0.3 0.9 0.570088 0.570088 -0.5 0.9 0.5 0.5 -0.5 0.9 0.707107 0.353553 -0.5 0.9 0.353553 0.707107 -0.5 0.9 0.866025 0.288675 -0.5 0.9 0.288661 0.866069 -0.5 0.9 0.570088 0.570088 -0.7 0.9 0.5 0.5 -0.7 0.9 0.707107 0.353553 -0.7 0.9 0.353553 0.707107 -0.7 0.9 0.866025 0.288675 -0.7 0.9 0.288661 0.866069 -0.7 0.9 0.570088 0.570088 -0.9 0.9 0.5 0.5 -0.9 0.9 0.707107 0.353553 -0.9 0.9 0.353553 0.707107 -0.9 0.9 0.866025 0.288675 -0.9 0.9 0.288661 0.866069 -0.9 0.9 0.570088 0.570088 -0.166667 0.166667 0.65 0.65 -0.166667 0.166667 0.919239 0.459619 -0.166667 0.166667 0.459619 0.919239 -0.166667 0.166667 1.12583 0.375278 -0.166667 0.166667 0.375259 1.12589 -0.166667 0.166667 0.72111 0.72111 -0.5 0.166667 0.65 0.65 -0.5 0.166667 0.919239 0.459619 -0.5 0.166667 0.459619 0.919239 -0.5 0.166667 1.12583 0.375278 -0.5 0.166667 0.375259 1.12589 -0.5 0.166667 0.72111 0.72111 -0.833333 0.166667 0.65 0.65 -0.833333 0.166667 0.919239 0.459619 -0.833333 0.166667 0.459619 0.919239 -0.833333 0.166667 1.12583 0.375278 -0.833333 0.166667 0.375259 1.12589 -0.833333 0.166667 0.72111 0.72111 -0.166667 0.5 0.65 0.65 -0.166667 0.5 0.919239 0.459619 -0.166667 0.5 0.459619 0.919239 -0.166667 0.5 1.12583 0.375278 -0.166667 0.5 0.375259 1.12589 -0.166667 0.5 0.72111 0.72111 -0.5 0.5 0.65 0.65 -0.5 0.5 0.919239 0.459619 -0.5 0.5 0.459619 0.919239 -0.5 0.5 1.12583 0.375278 -0.5 0.5 0.375259 1.12589 -0.5 0.5 0.72111 0.72111 -0.833333 0.5 0.65 0.65 -0.833333 0.5 0.919239 0.459619 -0.833333 0.5 0.459619 0.919239 -0.833333 0.5 1.12583 0.375278 -0.833333 0.5 0.375259 1.12589 -0.833333 0.5 0.72111 0.72111 -0.166667 0.833333 0.65 0.65 -0.166667 0.833333 0.919239 0.459619 -0.166667 0.833333 0.459619 0.919239 -0.166667 0.833333 1.12583 0.375278 -0.166667 0.833333 0.375259 1.12589 -0.166667 0.833333 0.72111 0.72111 -0.5 0.833333 0.65 0.65 -0.5 0.833333 0.919239 0.459619 -0.5 0.833333 0.459619 0.919239 -0.5 0.833333 1.12583 0.375278 -0.5 0.833333 0.375259 1.12589 -0.5 0.833333 0.72111 0.72111 -0.833333 0.833333 0.65 0.65 -0.833333 0.833333 0.919239 0.459619 -0.833333 0.833333 0.459619 0.919239 -0.833333 0.833333 1.12583 0.375278 -0.833333 0.833333 0.375259 1.12589 -0.833333 0.833333 0.72111 0.72111 -0.25 0.25 0.8 0.8 -0.25 0.25 1.13137 0.565686 -0.25 0.25 0.565685 1.13137 -0.25 0.25 1.38564 0.46188 -0.25 0.25 0.461857 1.38571 -0.25 0.25 0.87178 0.87178 -0.75 0.25 0.8 0.8 -0.75 0.25 1.13137 0.565686 -0.75 0.25 0.565685 1.13137 -0.75 0.25 1.38564 0.46188 -0.75 0.25 0.461857 1.38571 -0.75 0.25 0.87178 0.87178 -0.25 0.75 0.8 0.8 -0.25 0.75 1.13137 0.565686 -0.25 0.75 0.565685 1.13137 -0.25 0.75 1.38564 0.46188 -0.25 0.75 0.461857 1.38571 -0.25 0.75 0.87178 0.87178 -0.75 0.75 0.8 0.8 -0.75 0.75 1.13137 0.565686 -0.75 0.75 0.565685 1.13137 -0.75 0.75 1.38564 0.46188 -0.75 0.75 0.461857 1.38571 -0.75 0.75 0.87178 0.87178 -0.5 0.5 0.95 0.95 -0.5 0.5 1.3435 0.671751 -0.5 0.5 0.671751 1.3435 -0.5 0.5 1.64545 0.548483 -0.5 0.5 0.548455 1.64553 -0.5 0.5 0.974679 0.974679 diff --git a/mediapipe/calculators/tflite/testdata/labelmap.txt b/mediapipe/calculators/tflite/testdata/labelmap.txt deleted file mode 100644 index 4291e3c6b..000000000 --- a/mediapipe/calculators/tflite/testdata/labelmap.txt +++ /dev/null @@ -1,3 +0,0 @@ -classA -classB -classC diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator.cc b/mediapipe/calculators/tflite/tflite_converter_calculator.cc deleted file mode 100644 index d9dfd1526..000000000 --- a/mediapipe/calculators/tflite/tflite_converter_calculator.cc +++ /dev/null @@ -1,743 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" -#include "mediapipe/util/tflite/config.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gpu_buffer.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_program.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate.h" -#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE - -namespace { -constexpr int kWorkgroupSize = 8; // Block size for GPU shader. -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kTensorsGpuTag[] = "TENSORS_GPU"; -constexpr char kMatrixTag[] = "MATRIX"; -} // namespace - -namespace mediapipe { - -namespace { -#if MEDIAPIPE_TFLITE_GL_INFERENCE -using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer; -using ::tflite::gpu::gl::GlProgram; -using ::tflite::gpu::gl::GlShader; -struct GPUData { - int elements = 1; - GpuTensor buffer; - GlShader shader; - GlProgram program; -}; -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE -struct GPUData { - int elements = 1; - GpuTensor buffer; - id pipeline_state; -}; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -} // namespace - -// Calculator for normalizing and converting an ImageFrame or Matrix -// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer -// or MTLBuffer. -// -// This calculator is designed to be used with the TfLiteInferenceCalcualtor, -// as a pre-processing step for calculator inputs. -// -// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1], -// specified by options (unless outputting a quantized tensor). -// -// Input: -// One of the following tags: -// IMAGE - ImageFrame (assumed to be 8-bit or 32-bit data). -// IMAGE_GPU - GpuBuffer (assumed to be RGBA or RGB GL texture). -// MATRIX - Matrix. -// -// Output: -// One of the following tags: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32, or kTfLiteUint8. -// TENSORS_GPU - vector of GlBuffer or MTLBuffer. -// -// Example use: -// node { -// calculator: "TfLiteConverterCalculator" -// input_stream: "IMAGE:input_image" -// output_stream: "TENSORS:image_tensor" -// options: { -// [mediapipe.TfLiteConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -// IMPORTANT Notes: -// No conversion between CPU/GPU is done. -// Inputs/outputs must match type: CPU->CPU or GPU->GPU. -// GPU tensors are currently only supported on mobile platforms. -// This calculator uses FixedSizeInputStreamHandler by default. -// -// Note: Input defines output, so only these type sets are supported: -// IMAGE -> TENSORS | IMAGE_GPU -> TENSORS_GPU | MATRIX -> TENSORS -// -class TfLiteConverterCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status InitGpu(CalculatorContext* cc); - absl::Status LoadOptions(CalculatorContext* cc); - template - absl::Status NormalizeImage(const ImageFrame& image_frame, - bool flip_vertically, float* tensor_ptr); - absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr); - absl::Status ProcessCPU(CalculatorContext* cc); - absl::Status ProcessGPU(CalculatorContext* cc); - - std::unique_ptr interpreter_ = nullptr; - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - mediapipe::GlCalculatorHelper gpu_helper_; - std::unique_ptr gpu_data_out_; -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MPPMetalHelper* gpu_helper_ = nullptr; - std::unique_ptr gpu_data_out_; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - bool initialized_ = false; - bool use_gpu_ = false; - absl::optional> output_range_; - bool flip_vertically_ = false; - bool row_major_matrix_ = false; - bool use_quantized_tensors_ = false; - int max_num_channels_ = 3; -}; -REGISTER_CALCULATOR(TfLiteConverterCalculator); - -namespace { -template -bool ShouldUseGpu(CC* cc) { -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - return cc->Inputs().HasTag(kGpuBufferTag) || - cc->Outputs().HasTag(kTensorsGpuTag); -#else - return false; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED -} -} // namespace - -absl::Status TfLiteConverterCalculator::GetContract(CalculatorContract* cc) { - // Confirm only one of the input streams is present. - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^ - cc->Inputs().HasTag(kGpuBufferTag) ^ - cc->Inputs().HasTag(kMatrixTag)); - - // Confirm only one of the output streams is present. - RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^ - cc->Outputs().HasTag(kTensorsGpuTag)); - - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } - if (cc->Inputs().HasTag(kMatrixTag)) { - cc->Inputs().Tag(kMatrixTag).Set(); - } -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - cc->Inputs().Tag(kGpuBufferTag).Set(); - } -#endif // !MEDIAPIPE_DISABLE_GPU - - if (cc->Outputs().HasTag(kTensorsTag)) { - cc->Outputs().Tag(kTensorsTag).Set>(); - } - if (cc->Outputs().HasTag(kTensorsGpuTag)) { - cc->Outputs().Tag(kTensorsGpuTag).Set>(); - } - - if (ShouldUseGpu(cc)) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - - // Assign this calculator's default InputStreamHandler. - cc->SetInputStreamHandler("FixedSizeInputStreamHandler"); - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - use_gpu_ = ShouldUseGpu(cc); - - if (use_gpu_) { - // Cannot mix CPU/GPU streams. - RET_CHECK(cc->Inputs().HasTag(kGpuBufferTag) && - cc->Outputs().HasTag(kTensorsGpuTag)); - // Cannot use quantization. - use_quantized_tensors_ = false; -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } else { - interpreter_ = absl::make_unique(); - interpreter_->AddTensors(1); - interpreter_->SetInputs({0}); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::Process(CalculatorContext* cc) { - if (use_gpu_) { - if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) { - return absl::OkStatus(); - } - if (!initialized_) { - MP_RETURN_IF_ERROR(InitGpu(cc)); - initialized_ = true; - } - // Convert to GPU tensors type. - MP_RETURN_IF_ERROR(ProcessGPU(cc)); - } else { - // Convert to CPU tensors or Matrix type. - MP_RETURN_IF_ERROR(ProcessCPU(cc)); - } - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::Close(CalculatorContext* cc) { - interpreter_.reset(); -#if MEDIAPIPE_TFLITE_GL_INFERENCE - gpu_helper_.RunInGlContext([this] { gpu_data_out_.reset(); }); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - gpu_data_out_.reset(); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::ProcessCPU(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kImageFrameTag)) { - if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { - return absl::OkStatus(); - } - // CPU ImageFrame to TfLiteTensor conversion. - - const auto& image_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const mediapipe::ImageFormat::Format format = image_frame.Format(); - - if (!initialized_) { - if (!(format == mediapipe::ImageFormat::SRGBA || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::VEC32F1)) - RET_CHECK_FAIL() << "Unsupported CPU input format."; - TfLiteQuantization quant; - if (use_quantized_tensors_) { - RET_CHECK(format != mediapipe::ImageFormat::VEC32F1) - << "Only 8-bit input images are supported for quantization."; - quant.type = kTfLiteAffineQuantization; - auto quant_params = static_cast( - malloc(sizeof(TfLiteAffineQuantization))); - quant_params->scale = TfLiteFloatArrayCreate(1); - quant_params->scale->data[0] = 1.0; - quant_params->zero_point = TfLiteIntArrayCreate(1); - quant_params->zero_point->data[0] = 0; - quant_params->quantized_dimension = 0; - quant.params = quant_params; - interpreter_->SetTensorParametersReadWrite(0, kTfLiteUInt8, "", - {channels_preserved}, quant); - } else { - // Initialize structure for no quantization. - quant.type = kTfLiteNoQuantization; - quant.params = nullptr; - interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", - {channels_preserved}, quant); - } - initialized_ = true; - } - - const int tensor_idx = interpreter_->inputs()[0]; - TfLiteTensor* tensor = interpreter_->tensor(tensor_idx); - interpreter_->ResizeInputTensor(tensor_idx, - {height, width, channels_preserved}); - interpreter_->AllocateTensors(); - - // Copy image data into tensor. - if (use_quantized_tensors_) { - const int width_padding = - image_frame.WidthStep() / image_frame.ByteDepth() - width * channels; - const uint8* image_buffer = - reinterpret_cast(image_frame.PixelData()); - uint8* tensor_buffer = tensor->data.uint8; - RET_CHECK(tensor_buffer); - for (int row = 0; row < height; ++row) { - for (int col = 0; col < width; ++col) { - for (int channel = 0; channel < channels_preserved; ++channel) { - *tensor_buffer++ = image_buffer[channel]; - } - image_buffer += channels; - } - image_buffer += width_padding; - } - } else { - float* tensor_buffer = tensor->data.f; - RET_CHECK(tensor_buffer); - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - tensor_buffer)); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - tensor_buffer)); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } - } - - auto output_tensors = absl::make_unique>(); - output_tensors->emplace_back(*tensor); - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); - } else if (cc->Inputs().HasTag(kMatrixTag)) { - if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { - return absl::OkStatus(); - } - // CPU Matrix to TfLiteTensor conversion. - const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get(); - const int height = matrix.rows(); - const int width = matrix.cols(); - const int channels = 1; - - if (!initialized_) { - interpreter_->SetTensorParametersReadWrite( - /*tensor_index=*/0, /*type=*/kTfLiteFloat32, /*name=*/"", - /*dims=*/{channels}, /*quantization=*/TfLiteQuantization()); - initialized_ = true; - } - - const int tensor_idx = interpreter_->inputs()[0]; - TfLiteTensor* tensor = interpreter_->tensor(tensor_idx); - interpreter_->ResizeInputTensor(tensor_idx, {height, width, channels}); - interpreter_->AllocateTensors(); - - float* tensor_ptr = tensor->data.f; - RET_CHECK(tensor_ptr); - - MP_RETURN_IF_ERROR(CopyMatrixToTensor(matrix, tensor_ptr)); - - auto output_tensors = absl::make_unique>(); - output_tensors->emplace_back(*tensor); - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::ProcessGPU(CalculatorContext* cc) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - // GpuBuffer to tflite::gpu::GlBuffer conversion. - const auto& input = - cc->Inputs().Tag(kGpuBufferTag).Get(); - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, &input]() -> absl::Status { - // Convert GL texture into TfLite GlBuffer (SSBO). - auto src = gpu_helper_.CreateSourceTexture(input); - glActiveTexture(GL_TEXTURE0 + 0); - glBindTexture(GL_TEXTURE_2D, src.name()); - MP_RETURN_IF_ERROR(gpu_data_out_->buffer.BindToIndex(1)); - const tflite::gpu::uint3 workgroups = { - NumGroups(input.width(), kWorkgroupSize), - NumGroups(input.height(), kWorkgroupSize), 1}; - MP_RETURN_IF_ERROR(gpu_data_out_->program.Dispatch(workgroups)); - glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); - glBindTexture(GL_TEXTURE_2D, 0); - src.Release(); - return absl::OkStatus(); - })); - - // Copy into outputs. - auto output_tensors = absl::make_unique>(); - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, &output_tensors]() -> absl::Status { - output_tensors->resize(1); - { - GpuTensor& tensor = output_tensors->at(0); - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - gpu_data_out_->elements, &tensor)); - MP_RETURN_IF_ERROR(CopyBuffer(gpu_data_out_->buffer, tensor)); - } - return absl::OkStatus(); - })); - cc->Outputs() - .Tag(kTensorsGpuTag) - .Add(output_tensors.release(), cc->InputTimestamp()); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - // GpuBuffer to id conversion. - const auto& input = - cc->Inputs().Tag(kGpuBufferTag).Get(); - id command_buffer = [gpu_helper_ commandBuffer]; - - id src_texture = [gpu_helper_ metalTextureWithGpuBuffer:input]; - command_buffer.label = @"TfLiteConverterCalculatorConvertAndBlit"; - id compute_encoder = - [command_buffer computeCommandEncoder]; - [compute_encoder setComputePipelineState:gpu_data_out_->pipeline_state]; - [compute_encoder setTexture:src_texture atIndex:0]; - [compute_encoder setBuffer:gpu_data_out_->buffer offset:0 atIndex:1]; - MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, kWorkgroupSize, 1); - MTLSize threadgroups = - MTLSizeMake(NumGroups(input.width(), kWorkgroupSize), - NumGroups(input.height(), kWorkgroupSize), 1); - [compute_encoder dispatchThreadgroups:threadgroups - threadsPerThreadgroup:threads_per_group]; - [compute_encoder endEncoding]; - - // Copy into outputs. - // TODO Avoid this copy. - auto output_tensors = absl::make_unique>(); - output_tensors->resize(1); - id device = gpu_helper_.mtlDevice; - output_tensors->at(0) = - [device newBufferWithLength:gpu_data_out_->elements * sizeof(float) - options:MTLResourceStorageModeShared]; - [MPPMetalUtil blitMetalBufferTo:output_tensors->at(0) - from:gpu_data_out_->buffer - blocking:false - commandBuffer:command_buffer]; - - cc->Outputs() - .Tag(kTensorsGpuTag) - .Add(output_tensors.release(), cc->InputTimestamp()); -#else - RET_CHECK_FAIL() << "GPU processing is not enabled."; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::InitGpu(CalculatorContext* cc) { -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - // Get input image sizes. - const auto& input = - cc->Inputs().Tag(kGpuBufferTag).Get(); - mediapipe::ImageFormat::Format format = - mediapipe::ImageFormatForGpuBufferFormat(input.format()); - gpu_data_out_ = absl::make_unique(); - gpu_data_out_->elements = input.height() * input.width() * max_num_channels_; - const bool include_alpha = (max_num_channels_ == 4); - const bool single_channel = (max_num_channels_ == 1); - if (!(format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Unsupported GPU input format."; - if (include_alpha && (format != mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Num input channels is less than desired output."; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext( - [this, &include_alpha, &input, &single_channel]() -> absl::Status { - // Device memory. - MP_RETURN_IF_ERROR( - ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer( - gpu_data_out_->elements, &gpu_data_out_->buffer)); - - // Shader to convert GL Texture to Shader Storage Buffer Object (SSBO), - // with normalization to either: [0,1] or [-1,1]. - const std::string shader_source = absl::Substitute( - R"( #version 310 es - layout(local_size_x = $0, local_size_y = $0) in; - layout(binding = 0) uniform sampler2D input_texture; - layout(std430, binding = 1) buffer Output {float elements[];} output_data; - ivec2 width_height = ivec2($1, $2); - void main() { - ivec2 gid = ivec2(gl_GlobalInvocationID.xy); - if (gid.x >= width_height.x || gid.y >= width_height.y) return; - vec4 pixel = texelFetch(input_texture, gid, 0); - $3 // normalize [-1,1] - int linear_index = $7 * ($4 * width_height.x + gid.x); - output_data.elements[linear_index + 0] = pixel.x; // r channel - $5 // g & b channels - $6 // alpha channel - })", - /*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(), - /*$3=*/ - output_range_.has_value() - ? absl::Substitute( - "pixel = pixel * float($0) + float($1);", - (output_range_->second - output_range_->first), - output_range_->first) - : "", - /*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y", - /*$5=*/ - single_channel - ? "" - : R"(output_data.elements[linear_index + 1] = pixel.y; - output_data.elements[linear_index + 2] = pixel.z;)", - /*$6=*/ - include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;" - : "", - /*$7=*/max_num_channels_); - MP_RETURN_IF_ERROR(GlShader::CompileShader( - GL_COMPUTE_SHADER, shader_source, &gpu_data_out_->shader)); - MP_RETURN_IF_ERROR(GlProgram::CreateWithShader( - gpu_data_out_->shader, &gpu_data_out_->program)); - return absl::OkStatus(); - })); - -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - - RET_CHECK(include_alpha) - << "iOS GPU inference currently accepts only RGBA input."; - - // Device memory. - id device = gpu_helper_.mtlDevice; - gpu_data_out_->buffer = - [device newBufferWithLength:gpu_data_out_->elements * sizeof(float) - options:MTLResourceStorageModeShared]; - - // Shader to convert GL Texture to Metal Buffer, - // with normalization to either: [0,1] or [-1,1]. - const std::string shader_source = absl::Substitute( - R"( - #include - - using namespace metal; - - kernel void convertKernel( - texture2d in_tex [[ texture(0) ]], - device float* out_buf [[ buffer(1) ]], - uint2 gid [[ thread_position_in_grid ]]) { - if (gid.x >= in_tex.get_width() || gid.y >= in_tex.get_height()) return; - constexpr sampler texture_sampler(coord::pixel, address::clamp_to_edge); - const float2 coord = float2(gid.x, gid.y); - $0 pixel = $0(in_tex.sample(texture_sampler, coord).$1); - $2 // normalize [-1,1] - const int linear_index = $4 * ($3 * in_tex.get_width() + gid.x); - out_buf[linear_index + 0] = pixel.x; - out_buf[linear_index + 1] = pixel.y; - out_buf[linear_index + 2] = pixel.z; - $5 // alpha channel - } - )", - /*$0=*/include_alpha ? "float4" : "float3", - /*$1=*/include_alpha ? "rgba" : "rgb", - /*$2=*/ - output_range_.has_value() - ? absl::Substitute("pixel = pixel * float($0) + float($1);", - (output_range_->second - output_range_->first), - output_range_->first) - : "", - /*$3=*/flip_vertically_ ? "(in_tex.get_height() - 1 - gid.y)" : "gid.y", - /*$4=*/include_alpha ? 4 : 3, - /*$5=*/include_alpha ? "out_buf[linear_index + 3] = pixel.w;" : ""); - - NSString* library_source = - [NSString stringWithUTF8String:shader_source.c_str()]; - NSError* error = nil; - id library = - [device newLibraryWithSource:library_source options:nullptr error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"convertKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - gpu_data_out_->pipeline_state = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(gpu_data_out_->pipeline_state != nil) - << "Couldn't create pipeline state " - << [[error localizedDescription] UTF8String]; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::LoadOptions(CalculatorContext* cc) { - // Get calculator options specified in the graph. - const auto& options = - cc->Options<::mediapipe::TfLiteConverterCalculatorOptions>(); - - // if zero_center, set output float range to match [-1, 1] as specified in - // calculator proto. - if (options.zero_center()) { - output_range_.emplace(std::pair(-1.0, 1.0)); - } - - // Custom output_tensor_float_range values. - // If the float range is specified in pb text, use the specified values - // instead. - if (options.has_output_tensor_float_range()) { - output_range_.emplace(options.output_tensor_float_range().min(), - options.output_tensor_float_range().max()); - CHECK_GT(output_range_->second, output_range_->first); - } - - // Custom div and sub values. - if (options.use_custom_normalization()) { - output_range_.emplace(std::pair( - -options.custom_sub(), - -options.custom_sub() + 255.0 / options.custom_div())); - } - - // Get y-flip mode. - flip_vertically_ = options.flip_vertically(); - - // Get row_major_matrix mode. - row_major_matrix_ = options.row_major_matrix(); - - // Get desired way to handle input channels. - max_num_channels_ = options.max_num_channels(); - CHECK_GE(max_num_channels_, 1); - CHECK_LE(max_num_channels_, 4); - CHECK_NE(max_num_channels_, 2); -#if defined(MEDIAPIPE_IOS) - if (cc->Inputs().HasTag(kGpuBufferTag)) - // Currently on iOS, tflite gpu input tensor must be 4 channels, - // so input image must be 4 channels also (checked in InitGpu). - max_num_channels_ = 4; -#endif - - // Get tensor type, float or quantized. - use_quantized_tensors_ = options.use_quantized_tensors(); - - return absl::OkStatus(); -} - -template -absl::Status TfLiteConverterCalculator::NormalizeImage( - const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const int channels_ignored = channels - channels_preserved; - - if (output_range_.has_value()) { - // If the output float range is set and we are not using custom - // normalization, normalize the pixel values from [0, 255] to the specified - // output range. - RET_CHECK_NE(output_range_->first, output_range_->second); - const float scale = (output_range_->second - output_range_->first) / 255.0f; - const float bias = output_range_->first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - } else { - // [0,1], scale only (bias == 0) - // Verified that there are no precision issues with 1.0f / 255.0f expression - const float scale = 1.0f / 255.0f; - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale; - } - image_ptr += channels_ignored; - } - } - } - - return absl::OkStatus(); -} - -absl::Status TfLiteConverterCalculator::CopyMatrixToTensor(const Matrix& matrix, - float* tensor_ptr) { - if (row_major_matrix_) { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator.proto b/mediapipe/calculators/tflite/tflite_converter_calculator.proto deleted file mode 100644 index 5ed70879d..000000000 --- a/mediapipe/calculators/tflite/tflite_converter_calculator.proto +++ /dev/null @@ -1,69 +0,0 @@ -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "TfLiteConverterCalculator" -// input_stream: "IMAGE_IN:input_image" -// output_stream: "TENSOR_OUT:image_tensor" -// options { -// [mediapipe.TfLiteConverterCalculatorOptions.ext] { -// zero_center: true -// } -// } -// } -// -message TfLiteConverterCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TfLiteConverterCalculatorOptions ext = 245817797; - } - - // Choose normalization mode for output (not applied for Matrix inputs). - // true = [-1,1] - // false = [0,1] - // Ignored if using quantization. - optional bool zero_center = 1 [default = true]; - - // Custom settings to override the internal scaling factors `div` and `sub`. - // Both values must be set to non-negative values. Will only take effect on - // CPU AND when |use_custom_normalization| is set to true. When these custom - // values take effect, the |zero_center| setting above will be overriden, and - // the normalized_value will be calculated as: - // normalized_value = input / custom_div - custom_sub. - optional bool use_custom_normalization = 6 [default = false]; - optional float custom_div = 7 [default = -1.0]; - optional float custom_sub = 8 [default = -1.0]; - - // Whether the input image should be flipped vertically (along the - // y-direction). This is useful, for example, when the input image is defined - // with a coordinate system where the origin is at the bottom-left corner - // (e.g., in OpenGL) whereas the ML model expects an image with a top-left - // origin. - optional bool flip_vertically = 2 [default = false]; - - // Controls how many channels of the input image get passed through to the - // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU. - optional int32 max_num_channels = 3 [default = 3]; - - // The calculator expects Matrix inputs to be in column-major order. Set - // row_major_matrix to true if the inputs are in row-major order. - optional bool row_major_matrix = 4 [default = false]; - - // Quantization option (CPU only). - // When true, output kTfLiteUInt8 tensor instead of kTfLiteFloat32. - optional bool use_quantized_tensors = 5 [default = false]; - - // Normalization option. - // Setting normalization_range results in the values normalized to - // the range [output_tensor_float_range.min, output_tensor_float_range.max]. - optional TensorFloatRange output_tensor_float_range = 9; - - message TensorFloatRange { - optional float min = 1; - optional float max = 2; - } -} diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator_test.cc b/mediapipe/calculators/tflite/tflite_converter_calculator_test.cc deleted file mode 100644 index aab62901d..000000000 --- a/mediapipe/calculators/tflite/tflite_converter_calculator_test.cc +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/substitute.h" -#include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { -namespace { - -constexpr char kTransposeOptionsString[] = - "[mediapipe.TfLiteConverterCalculatorOptions.ext]: {" - "row_major_matrix: True}"; - -} // namespace - -using RandomEngine = std::mt19937_64; -using testing::Eq; -const uint32 kSeed = 1234; -const int kNumSizes = 8; -const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, - {5, 3}, {7, 13}, {16, 32}, {101, 2}}; - -class TfLiteConverterCalculatorTest : public ::testing::Test { - protected: - // Adds a packet with a matrix filled with random values in [0,1]. - void AddRandomMatrix(int num_rows, int num_columns, uint32 seed, - bool row_major_matrix = false) { - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - auto matrix = ::absl::make_unique(); - matrix->resize(num_rows, num_columns); - if (row_major_matrix) { - for (int y = 0; y < num_rows; ++y) { - for (int x = 0; x < num_columns; ++x) { - float value = uniform_dist(random); - (*matrix)(y, x) = value; - } - } - } else { - for (int x = 0; x < num_columns; ++x) { - for (int y = 0; y < num_rows; ++y) { - float value = uniform_dist(random); - (*matrix)(y, x) = value; - } - } - } - MP_ASSERT_OK(graph_->AddPacketToInputStream( - "matrix", Adopt(matrix.release()).At(Timestamp(0)))); - } - - std::unique_ptr graph_; -}; - -TEST_F(TfLiteConverterCalculatorTest, RandomMatrixColMajor) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "matrix" - node { - calculator: "TfLiteConverterCalculator" - input_stream: "MATRIX:matrix" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - row_major_matrix: false - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - graph_ = absl::make_unique(); - MP_ASSERT_OK(graph_->Initialize(graph_config)); - MP_ASSERT_OK(graph_->StartRun({})); - - // Push the tensor into the graph. - AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/false); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const TfLiteTensor* tensor = &tensor_vec[0]; - EXPECT_EQ(kTfLiteFloat32, tensor->type); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - const float* tensor_buffer = tensor->data.f; - for (int i = 0; i < num_rows * num_columns; ++i) { - const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph_->CloseInputStream("matrix")); - MP_ASSERT_OK(graph_->WaitUntilDone()); - - graph_.reset(); - } -} - -TEST_F(TfLiteConverterCalculatorTest, RandomMatrixRowMajor) { - for (int size_index = 0; size_index < kNumSizes; ++size_index) { - const int num_rows = sizes[size_index][0]; - const int num_columns = sizes[size_index][1]; - - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "matrix" - node { - calculator: "TfLiteConverterCalculator" - input_stream: "MATRIX:matrix" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - row_major_matrix: true - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - graph_ = absl::make_unique(); - MP_ASSERT_OK(graph_->Initialize(graph_config)); - MP_ASSERT_OK(graph_->StartRun({})); - - // Push the tensor into the graph. - AddRandomMatrix(num_rows, num_columns, kSeed, /*row_major_matrix=*/true); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const TfLiteTensor* tensor = &tensor_vec[0]; - EXPECT_EQ(kTfLiteFloat32, tensor->type); - - // Verify that the data is correct. - RandomEngine random(kSeed); - std::uniform_real_distribution<> uniform_dist(0, 1.0); - const float* tensor_buffer = tensor->data.f; - for (int i = 0; i < num_rows * num_columns; ++i) { - const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph_->CloseInputStream("matrix")); - MP_ASSERT_OK(graph_->WaitUntilDone()); - - graph_.reset(); - } -} - -TEST_F(TfLiteConverterCalculatorTest, CustomDivAndSub) { - CalculatorGraph graph; - // Run the calculator and verify that one output is generated. - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "input_image" - node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - row_major_matrix: true - use_custom_normalization: true - custom_div: 2.0 - custom_sub: 33.0 - } - } - } - )pb"); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); - - const TfLiteTensor* tensor = &tensor_vec[0]; - EXPECT_EQ(kTfLiteFloat32, tensor->type); - EXPECT_FLOAT_EQ(67.0f, *tensor->data.f); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -TEST_F(TfLiteConverterCalculatorTest, SetOutputRange) { - std::vector> range_values = { - std::make_pair(0.0, 1.0), std::make_pair(-1.0, 1.0), - std::make_pair(-0.5, 0.5)}; - for (std::pair range : range_values) { - CalculatorGraph graph; - CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "input_image" - node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - output_tensor_float_range { - min: $0 - max: $1 - } - } - } - } - )", - /*$0=*/range.first, - /*$1=*/range.second)); - std::vector output_packets; - tool::AddVectorSink("tensor", &graph_config, &output_packets); - - // Run the graph. - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 1); - cv::Mat mat = mediapipe::formats::MatView(input_image.get()); - mat.at(0, 0) = 200; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_image", Adopt(input_image.release()).At(Timestamp(0)))); - - // Wait until the calculator finishes processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets.size(), Eq(1)); - - // Get and process results. - const std::vector& tensor_vec = - output_packets[0].Get>(); - EXPECT_THAT(tensor_vec.size(), Eq(1)); - - const TfLiteTensor* tensor = &tensor_vec[0]; - - // Calculate the expected normalized value: - float normalized_value = - range.first + (200 * (range.second - range.first)) / 255.0; - - EXPECT_THAT(tensor->type, Eq(kTfLiteFloat32)); - EXPECT_THAT(normalized_value, - testing::FloatNear(*tensor->data.f, - 2.0f * std::abs(*tensor->data.f) * - std::numeric_limits::epsilon())); - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("input_image")); - MP_ASSERT_OK(graph.WaitUntilDone()); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.cc b/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.cc deleted file mode 100644 index 11e27dff1..000000000 --- a/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/tflite/cpu_op_resolver.h" -#include "mediapipe/util/tflite/op_resolver.h" - -namespace mediapipe { - -// This calculator creates a custom op resolver as a side packet that can be -// used in TfLiteInferenceCalculator. Current custom op resolver supports the -// following custom op on CPU and GPU: -// Convolution2DTransposeBias -// MaxPoolArgmax -// MaxUnpooling -// -// Usage example: -// node { -// calculator: "TfLiteCustomOpResolverCalculator" -// output_side_packet: "op_resolver" -// node_options: { -// [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] { -// use_gpu: true -// } -// } -// } -class TfLiteCustomOpResolverCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->OutputSidePackets() - .Index(0) - .Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - const TfLiteCustomOpResolverCalculatorOptions& options = - cc->Options(); - - std::unique_ptr op_resolver; - if (options.use_gpu()) { - op_resolver = absl::make_unique(); - } else { - op_resolver = absl::make_unique(); - } - - cc->OutputSidePackets().Index(0).Set(Adopt(op_resolver.release())); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(TfLiteCustomOpResolverCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.proto b/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.proto deleted file mode 100644 index 546165a9f..000000000 --- a/mediapipe/calculators/tflite/tflite_custom_op_resolver_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Options to generate an op resolver for running TfLite inference. -message TfLiteCustomOpResolverCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TfLiteCustomOpResolverCalculatorOptions ext = 252087553; - } - - // Flag for using GPU inference which uses the correspondent op resolver. - optional bool use_gpu = 1 [default = false]; -} diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc deleted file mode 100644 index 9ec556987..000000000 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ /dev/null @@ -1,1114 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/tflite/config.h" - -#if !defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__) -#include "mediapipe/util/cpu_util.h" -#endif // !__EMSCRIPTEN__ || __EMSCRIPTEN_PTHREADS__ - -#include "mediapipe/util/tflite/tflite_model_loader.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" - -#if defined(MEDIAPIPE_ANDROID) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/filesystem.h" -#include "mediapipe/util/android/file/base/helpers.h" -#endif // ANDROID - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/util/tflite/tflite_gpu_runner.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_program.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "tensorflow/lite/delegates/gpu/common/shape.h" -#include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h" -#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE - -#if !defined(MEDIAPIPE_EDGE_TPU) -#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h" -#endif // !EDGETPU -#if defined(MEDIAPIPE_ANDROID) -#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" -#endif // ANDROID - -namespace { -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -// Round up n to next multiple of m. -size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT - -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kTensorsGpuTag[] = "TENSORS_GPU"; -} // namespace - -#if defined(MEDIAPIPE_EDGE_TPU) -#include "edgetpu.h" - -// Creates and returns an Edge TPU interpreter to run the given edgetpu model. -std::unique_ptr BuildEdgeTpuInterpreter( - const tflite::FlatBufferModel& model, - tflite::ops::builtin::BuiltinOpResolver* resolver, - edgetpu::EdgeTpuContext* edgetpu_context) { - resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); - std::unique_ptr interpreter; - if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) { - std::cerr << "Failed to build edge TPU interpreter." << std::endl; - } - interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context); - interpreter->SetNumThreads(1); - if (interpreter->AllocateTensors() != kTfLiteOk) { - std::cerr << "Failed to allocate edge TPU tensors." << std::endl; - } - return interpreter; -} -#endif // MEDIAPIPE_EDGE_TPU - -// TfLiteInferenceCalculator File Layout: -// * Header -// * Core -// * Aux -namespace mediapipe { - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -using ::tflite::gpu::gl::CopyBuffer; -using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer; -using ::tflite::gpu::gl::GlBuffer; -#endif - -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED -namespace { -struct GPUData { - int elements = 1; - GpuTensor buffer; - ::tflite::gpu::BHWC shape; -}; -} // namespace -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED - -namespace { - -int GetXnnpackDefaultNumThreads() { -#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_IOS) || \ - defined(__EMSCRIPTEN_PTHREADS__) - constexpr int kMinNumThreadsByDefault = 1; - constexpr int kMaxNumThreadsByDefault = 4; - return std::clamp(NumCPUCores() / 2, kMinNumThreadsByDefault, - kMaxNumThreadsByDefault); -#else - return 1; -#endif // MEDIAPIPE_ANDROID || MEDIAPIPE_IOS || __EMSCRIPTEN_PTHREADS__ -} - -// Returns number of threads to configure XNNPACK delegate with. -// Returns user provided value if specified. Otherwise, tries to choose optimal -// number of threads depending on the device. -int GetXnnpackNumThreads( - const mediapipe::TfLiteInferenceCalculatorOptions& opts) { - static constexpr int kDefaultNumThreads = -1; - if (opts.has_delegate() && opts.delegate().has_xnnpack() && - opts.delegate().xnnpack().num_threads() != kDefaultNumThreads) { - return opts.delegate().xnnpack().num_threads(); - } - return GetXnnpackDefaultNumThreads(); -} - -} // namespace - -// Calculator Header Section - -// Runs inference on the provided input TFLite tensors and TFLite model. -// -// Creates an interpreter with given model and calls invoke(). -// Optionally run inference on CPU/GPU. -// -// This calculator is designed to be used with the TfLiteConverterCalculator, -// to get the appropriate inputs. -// -// When the input tensors are on CPU, gpu inference is optional and can be -// specified in the calculator options. -// When the input tensors are on GPU, inference is GPU and output can be CPU or -// GPU. -// -// Input: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 or kTfLiteUInt8 -// TENSORS_GPU - Vector of GlBuffer or MTLBuffer -// -// Output: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 or kTfLiteUInt8 -// TENSORS_GPU - Vector of GlBuffer or MTLBuffer -// -// Input side packet: -// CUSTOM_OP_RESOLVER (optional) - Use a custom op resolver, -// instead of the builtin one. -// MODEL (optional) - Use to specify TfLite model -// (std::unique_ptr>) -// -// Example use: -// node { -// calculator: "TfLiteInferenceCalculator" -// input_stream: "TENSORS:tensor_image" -// output_stream: "TENSORS:tensors" -// options: { -// [mediapipe.TfLiteInferenceCalculatorOptions.ext] { -// model_path: "modelname.tflite" -// } -// } -// } -// -// or -// -// node { -// calculator: "TfLiteInferenceCalculator" -// input_stream: "TENSORS_GPU:tensor_image" -// input_side_packet: "MODEL:model" -// output_stream: "TENSORS_GPU:tensors" -// options: { -// [mediapipe.TfLiteInferenceCalculatorOptions.ext] { -// model_path: "modelname.tflite" -// delegate { gpu {} } -// } -// } -// } -// -// IMPORTANT Notes: -// Tensors are assumed to be ordered correctly (sequentially added to model). -// Input tensors are assumed to be of the correct size and already normalized. -// All output TfLiteTensors will be destroyed when the graph closes, -// (i.e. after calling graph.WaitUntilDone()). -// GPU tensor support rquires OpenGL ES 3.1+. -// This calculator uses FixedSizeInputStreamHandler by default. -// -class TfLiteInferenceCalculator : public CalculatorBase { - public: - using TfLiteDelegatePtr = - std::unique_ptr>; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status ReadKernelsFromFile(); - absl::Status WriteKernelsToFile(); - absl::Status LoadModel(CalculatorContext* cc); - absl::StatusOr GetModelAsPacket(const CalculatorContext& cc); - absl::Status LoadDelegate(CalculatorContext* cc); - absl::Status InitTFLiteGPURunner(CalculatorContext* cc); - absl::Status ProcessInputsCpu(CalculatorContext* cc, - std::vector* output_tensors_cpu); - absl::Status ProcessOutputsCpu( - CalculatorContext* cc, - std::unique_ptr> output_tensors_cpu); - absl::Status ProcessInputsGpu(CalculatorContext* cc, - std::vector* output_tensors_gpu); - absl::Status ProcessOutputsGpu( - CalculatorContext* cc, - std::unique_ptr> output_tensors_cpu, - std::unique_ptr> output_tensors_gpu); - - absl::Status RunInContextIfNeeded(std::function f) { - if (gpu_inference_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - return gpu_helper_.RunInGlContext(std::move(f)); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - return f(); - } - - Packet model_packet_; - std::unique_ptr interpreter_; - TfLiteDelegatePtr delegate_; - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - mediapipe::GlCalculatorHelper gpu_helper_; - std::vector> gpu_data_in_; - std::vector> gpu_data_out_; - std::unique_ptr tflite_gpu_runner_; -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MPPMetalHelper* gpu_helper_ = nullptr; - std::vector> gpu_data_in_; - std::vector> gpu_data_out_; - id fp32_to_fp16_program_; - TFLBufferConvert* converter_from_BPHWC4_ = nil; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if defined(MEDIAPIPE_EDGE_TPU) - std::shared_ptr edgetpu_context_ = - edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice(); -#endif - - bool gpu_inference_ = false; - bool gpu_input_ = false; - bool gpu_output_ = false; - bool use_quantized_tensors_ = false; - - bool use_advanced_gpu_api_ = false; - bool allow_precision_loss_ = false; - mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::Api - tflite_gpu_runner_api_; - - bool use_kernel_caching_ = false; - std::string cached_kernel_filename_; -}; -REGISTER_CALCULATOR(TfLiteInferenceCalculator); - -// Calculator Core Section - -namespace { -template -bool ShouldUseGpu(CC* cc) { -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - const auto& options = - cc->template Options<::mediapipe::TfLiteInferenceCalculatorOptions>(); - return options.use_gpu() || - (options.has_delegate() && options.delegate().has_gpu()) || - cc->Inputs().HasTag(kTensorsGpuTag) || - cc->Outputs().HasTag(kTensorsGpuTag); -#else - return false; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED -} -} // namespace - -absl::Status TfLiteInferenceCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kTensorsTag) ^ - cc->Inputs().HasTag(kTensorsGpuTag)); - RET_CHECK(cc->Outputs().HasTag(kTensorsTag) ^ - cc->Outputs().HasTag(kTensorsGpuTag)); - - const auto& options = - cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>(); - RET_CHECK(!options.model_path().empty() ^ - cc->InputSidePackets().HasTag("MODEL")) - << "Either model as side packet or model path in options is required."; - - if (cc->Inputs().HasTag(kTensorsTag)) - cc->Inputs().Tag(kTensorsTag).Set>(); - if (cc->Outputs().HasTag(kTensorsTag)) - cc->Outputs().Tag(kTensorsTag).Set>(); - - if (cc->Inputs().HasTag(kTensorsGpuTag)) - cc->Inputs().Tag(kTensorsGpuTag).Set>(); - if (cc->Outputs().HasTag(kTensorsGpuTag)) - cc->Outputs().Tag(kTensorsGpuTag).Set>(); - - if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) { - cc->InputSidePackets() - .Tag("CUSTOM_OP_RESOLVER") - .Set(); - } - if (cc->InputSidePackets().HasTag("MODEL")) { - cc->InputSidePackets().Tag("MODEL").Set(); - } - - if (ShouldUseGpu(cc)) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#endif - } - - // Assign this calculator's default InputStreamHandler. - cc->SetInputStreamHandler("FixedSizeInputStreamHandler"); - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = - cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>(); - - gpu_inference_ = ShouldUseGpu(cc); - gpu_input_ = cc->Inputs().HasTag(kTensorsGpuTag); - gpu_output_ = cc->Outputs().HasTag(kTensorsGpuTag); - - use_advanced_gpu_api_ = MEDIAPIPE_TFLITE_GL_INFERENCE && - options.has_delegate() && - options.delegate().has_gpu() && - options.delegate().gpu().use_advanced_gpu_api(); - allow_precision_loss_ = options.delegate().gpu().allow_precision_loss(); - tflite_gpu_runner_api_ = options.delegate().gpu().api(); - - use_kernel_caching_ = use_advanced_gpu_api_ && - options.delegate().gpu().has_cached_kernel_path(); - - if (use_kernel_caching_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID) - cached_kernel_filename_ = options.delegate().gpu().cached_kernel_path() + - mediapipe::File::Basename(options.model_path()) + - ".ker"; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID - } - - if (use_advanced_gpu_api_ && !gpu_input_) { - LOG(WARNING) << "Cannot use advanced GPU APIs, input must be GPU buffers." - "Falling back to the default TFLite API."; - use_advanced_gpu_api_ = false; - } - CHECK(!use_advanced_gpu_api_ || gpu_inference_); - - MP_RETURN_IF_ERROR(LoadModel(cc)); - - if (gpu_inference_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, - &cc]() -> absl::Status { - return use_advanced_gpu_api_ ? InitTFLiteGPURunner(cc) : LoadDelegate(cc); - })); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); - MP_RETURN_IF_ERROR(LoadDelegate(cc)); -#endif - } else { - MP_RETURN_IF_ERROR(LoadDelegate(cc)); - } - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::Process(CalculatorContext* cc) { - return RunInContextIfNeeded([this, cc]() -> absl::Status { - // 0. Declare outputs - auto output_tensors_gpu = absl::make_unique>(); - auto output_tensors_cpu = absl::make_unique>(); - - // 1. Receive pre-processed tensor inputs. - if (gpu_input_) { - MP_RETURN_IF_ERROR(ProcessInputsGpu(cc, output_tensors_gpu.get())); - } else { - MP_RETURN_IF_ERROR(ProcessInputsCpu(cc, output_tensors_cpu.get())); - } - - // 2. Run inference. -#if MEDIAPIPE_TFLITE_GL_INFERENCE - if (gpu_inference_ && use_advanced_gpu_api_) { - RET_CHECK(tflite_gpu_runner_->Invoke().ok()); - } else { - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - } -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - // Metal delegate supports external command buffer only if all input and - // output buffers are on GPU. - if (gpu_inference_ && gpu_input_ && gpu_output_) { - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"TfLiteInferenceCalculator"; - RET_CHECK( - TFLGpuDelegateSetCommandBuffer(delegate_.get(), command_buffer)); - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - [command_buffer commit]; - } else { - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); - } -#else // MEDIAPIPE_TFLITE_GL_INFERENCE - RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - // 3. Output processed tensors. - if (gpu_output_ || use_advanced_gpu_api_) { - MP_RETURN_IF_ERROR(ProcessOutputsGpu(cc, std::move(output_tensors_cpu), - std::move(output_tensors_gpu))); - } else { - MP_RETURN_IF_ERROR(ProcessOutputsCpu(cc, std::move(output_tensors_cpu))); - } - - return absl::OkStatus(); - }); -} - -absl::Status TfLiteInferenceCalculator::WriteKernelsToFile() { -#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID) - if (use_kernel_caching_) { - // Save kernel file. - auto kernel_cache = absl::make_unique>( - tflite_gpu_runner_->GetSerializedBinaryCache()); - std::string cache_str(kernel_cache->begin(), kernel_cache->end()); - MP_RETURN_IF_ERROR( - mediapipe::file::SetContents(cached_kernel_filename_, cache_str)); - } -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::Close(CalculatorContext* cc) { - MP_RETURN_IF_ERROR(WriteKernelsToFile()); - - return RunInContextIfNeeded([this]() -> absl::Status { - if (delegate_) { - interpreter_ = nullptr; - delegate_ = nullptr; -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED - if (gpu_inference_) { - for (int i = 0; i < gpu_data_in_.size(); ++i) { - gpu_data_in_[i].reset(); - } - for (int i = 0; i < gpu_data_out_.size(); ++i) { - gpu_data_out_[i].reset(); - } - } -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED - } -#if defined(MEDIAPIPE_EDGE_TPU) - edgetpu_context_.reset(); -#endif - return absl::OkStatus(); - }); -} - -// Calculator Auxiliary Section - -absl::Status TfLiteInferenceCalculator::ProcessInputsCpu( - CalculatorContext* cc, std::vector* output_tensors_cpu) { - if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) { - return absl::OkStatus(); - } - // Read CPU input into tensors. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - RET_CHECK_GT(input_tensors.size(), 0); - for (int i = 0; i < input_tensors.size(); ++i) { - const TfLiteTensor* input_tensor = &input_tensors[i]; - RET_CHECK(input_tensor->data.raw); - if (use_quantized_tensors_) { - const uint8* input_tensor_buffer = input_tensor->data.uint8; - uint8* local_tensor_buffer = interpreter_->typed_input_tensor(i); - std::memcpy(local_tensor_buffer, input_tensor_buffer, - input_tensor->bytes); - } else { - const float* input_tensor_buffer = input_tensor->data.f; - float* local_tensor_buffer = interpreter_->typed_input_tensor(i); - std::memcpy(local_tensor_buffer, input_tensor_buffer, - input_tensor->bytes); - } - } - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::ProcessInputsGpu( - CalculatorContext* cc, std::vector* output_tensors_gpu) { - if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) { - return absl::OkStatus(); - } - if (use_advanced_gpu_api_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - RET_CHECK(!input_tensors.empty()); - for (int i = 0; i < input_tensors.size(); ++i) { - MP_RETURN_IF_ERROR( - tflite_gpu_runner_->BindSSBOToInputTensor(input_tensors[i].id(), i)); - } - if (gpu_output_) { - // Allocate new output tensor. - output_tensors_gpu->resize(gpu_data_out_.size()); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - GpuTensor& tensor = output_tensors_gpu->at(i); - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - gpu_data_out_[i]->elements, &tensor)); - MP_RETURN_IF_ERROR( - tflite_gpu_runner_->BindSSBOToOutputTensor(tensor.id(), i)); - } - } else { - // Re-use internal output tensor. - for (int i = 0; i < gpu_data_out_.size(); ++i) { - MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor( - gpu_data_out_[i]->buffer.id(), i)); - } - } -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } else if (gpu_input_) { - // Read GPU input into SSBO. -#if MEDIAPIPE_TFLITE_GL_INFERENCE - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - RET_CHECK_GT(input_tensors.size(), 0); - // Explicit copy input. - gpu_data_in_.resize(input_tensors.size()); - for (int i = 0; i < input_tensors.size(); ++i) { - MP_RETURN_IF_ERROR(CopyBuffer(input_tensors[i], gpu_data_in_[i]->buffer)); - } -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - RET_CHECK_GT(input_tensors.size(), 0); - // Explicit copy input with conversion float 32 bits to 16 bits. - gpu_data_in_.resize(input_tensors.size()); - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"TfLiteInferenceCalculatorConvert"; - id compute_encoder = - [command_buffer computeCommandEncoder]; - [compute_encoder setComputePipelineState:fp32_to_fp16_program_]; - for (int i = 0; i < input_tensors.size(); ++i) { - [compute_encoder setBuffer:input_tensors[i] offset:0 atIndex:0]; - [compute_encoder setBuffer:gpu_data_in_[i]->buffer offset:0 atIndex:1]; - constexpr int kWorkgroupSize = 64; // Block size for GPU shader. - MTLSize threads_per_group = MTLSizeMake(kWorkgroupSize, 1, 1); - const int threadgroups = - NumGroups(gpu_data_in_[i]->elements, kWorkgroupSize); - [compute_encoder dispatchThreadgroups:MTLSizeMake(threadgroups, 1, 1) - threadsPerThreadgroup:threads_per_group]; - } - [compute_encoder endEncoding]; - [command_buffer commit]; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::ProcessOutputsCpu( - CalculatorContext* cc, - std::unique_ptr> output_tensors_cpu) { - // Output result tensors (CPU). - const auto& tensor_indexes = interpreter_->outputs(); - for (int i = 0; i < tensor_indexes.size(); ++i) { - TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]); - output_tensors_cpu->emplace_back(*tensor); - } - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors_cpu.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::ProcessOutputsGpu( - CalculatorContext* cc, - std::unique_ptr> output_tensors_cpu, - std::unique_ptr> output_tensors_gpu) { - if (use_advanced_gpu_api_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - if (gpu_output_) { - // Send out pre-allocated tensors. - cc->Outputs() - .Tag(kTensorsGpuTag) - .Add(output_tensors_gpu.release(), cc->InputTimestamp()); - } else { - // Download to CPU for output. - const auto& tensor_indexes = interpreter_->inputs(); - for (int i = 0; i < tensor_indexes.size(); ++i) { - TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]); - std::vector gpu_data(tensor->bytes / sizeof(float)); - MP_RETURN_IF_ERROR(gpu_data_out_[i]->buffer.Read( - absl::MakeSpan(tensor->data.f, tensor->bytes))); - output_tensors_cpu->emplace_back(*tensor); - } - // Output result tensors (CPU). - cc->Outputs() - .Tag(kTensorsTag) - .Add(output_tensors_cpu.release(), cc->InputTimestamp()); - } -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } else if (gpu_output_) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - // Output result tensors (GPU). - output_tensors_gpu->resize(gpu_data_out_.size()); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - GpuTensor& tensor = output_tensors_gpu->at(i); - // Allocate output tensor. - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - gpu_data_out_[i]->elements, &tensor)); - MP_RETURN_IF_ERROR(CopyBuffer(gpu_data_out_[i]->buffer, tensor)); - } - cc->Outputs() - .Tag(kTensorsGpuTag) - .Add(output_tensors_gpu.release(), cc->InputTimestamp()); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - // Output result tensors (GPU). - output_tensors_gpu->resize(gpu_data_out_.size()); - id device = gpu_helper_.mtlDevice; - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"TfLiteInferenceBPHWC4Convert"; - id convert_command = - [command_buffer computeCommandEncoder]; - for (int i = 0; i < gpu_data_out_.size(); ++i) { - // Allocate output tensor. - output_tensors_gpu->at(i) = - [device newBufferWithLength:gpu_data_out_[i]->elements * sizeof(float) - options:MTLResourceStorageModeShared]; - // Reshape tensor. - [converter_from_BPHWC4_ convertWithEncoder:convert_command - shape:gpu_data_out_[i]->shape - sourceBuffer:gpu_data_out_[i]->buffer - convertedBuffer:output_tensors_gpu->at(i)]; - } - [convert_command endEncoding]; - [command_buffer commit]; - cc->Outputs() - .Tag(kTensorsGpuTag) - .Add(output_tensors_gpu.release(), cc->InputTimestamp()); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::ReadKernelsFromFile() { -#if MEDIAPIPE_TFLITE_GL_INFERENCE && defined(MEDIAPIPE_ANDROID) - if (use_kernel_caching_) { - // Load pre-compiled kernel file. - if (mediapipe::File::Exists(cached_kernel_filename_)) { - std::string cache_str; - MP_RETURN_IF_ERROR( - mediapipe::file::GetContents(cached_kernel_filename_, &cache_str)); - std::vector cache_vec(cache_str.begin(), cache_str.end()); - tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec)); - } - } -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE && MEDIAPIPE_ANDROID - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner( - CalculatorContext* cc) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc)); - const auto& model = *model_packet_.Get(); - - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates - default_op_resolver; - auto op_resolver_ptr = - static_cast( - &default_op_resolver); - if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) { - op_resolver_ptr = &(cc->InputSidePackets() - .Tag("CUSTOM_OP_RESOLVER") - .Get()); - } - - // Create runner - tflite::gpu::InferenceOptions options; - options.priority1 = allow_precision_loss_ - ? tflite::gpu::InferencePriority::MIN_LATENCY - : tflite::gpu::InferencePriority::MAX_PRECISION; - options.priority2 = tflite::gpu::InferencePriority::AUTO; - options.priority3 = tflite::gpu::InferencePriority::AUTO; - options.usage = tflite::gpu::InferenceUsage::SUSTAINED_SPEED; - tflite_gpu_runner_ = std::make_unique(options); - switch (tflite_gpu_runner_api_) { - case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::OPENGL: { - tflite_gpu_runner_->ForceOpenGL(); - break; - } - case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::OPENCL: { - tflite_gpu_runner_->ForceOpenCL(); - break; - } - case mediapipe::TfLiteInferenceCalculatorOptions::Delegate::Gpu::ANY: { - // Do not need to force any specific API. - break; - } - } - MP_RETURN_IF_ERROR(tflite_gpu_runner_->InitializeWithModel( - model, *op_resolver_ptr, /*allow_quant_ops=*/true)); - - // Allocate interpreter memory for cpu output. - if (!gpu_output_) { - interpreter_ = absl::make_unique(); - const int num_outputs = tflite_gpu_runner_->GetOutputShapes().size(); - interpreter_->AddTensors(num_outputs); - std::vector indices(num_outputs); - for (int i = 0; i < num_outputs; ++i) indices[i] = i; - // There is no ResizeOutputTensor(), so we use 'inputs' space instead. - interpreter_->SetInputs(indices); - TfLiteQuantization quant; - quant.type = kTfLiteNoQuantization; - quant.params = nullptr; - for (int i = 0; i < num_outputs; ++i) { - auto shape = tflite_gpu_runner_->GetTFLiteOutputShapes()[i]; - const int tensor_idx = interpreter_->inputs()[i]; - interpreter_->SetTensorParametersReadWrite(tensor_idx, kTfLiteFloat32, "", - shape, quant); - CHECK(interpreter_->ResizeInputTensor(tensor_idx, shape) == kTfLiteOk); - } - CHECK(interpreter_->AllocateTensors() == kTfLiteOk); - } - - // Create and bind OpenGL buffers for outputs. - // The buffers are created once and their ids are passed to calculator outputs - gpu_data_out_.resize(tflite_gpu_runner_->outputs_size()); - for (int i = 0; i < tflite_gpu_runner_->outputs_size(); ++i) { - gpu_data_out_[i] = absl::make_unique(); - ASSIGN_OR_RETURN(gpu_data_out_[i]->elements, - tflite_gpu_runner_->GetOutputElements(i)); - // Create and bind input buffer. - MP_RETURN_IF_ERROR( - ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer( - gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer)); - } - - MP_RETURN_IF_ERROR(ReadKernelsFromFile()); - - MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build()); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - return absl::OkStatus(); -} - -absl::Status TfLiteInferenceCalculator::LoadModel(CalculatorContext* cc) { - if (use_advanced_gpu_api_) { - // Use InitTFLiteGPURunner for everything. - return absl::OkStatus(); - } - - ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(*cc)); - const auto& model = *model_packet_.Get(); - - tflite::ops::builtin::BuiltinOpResolverWithoutDefaultDelegates - default_op_resolver; - auto op_resolver_ptr = - static_cast( - &default_op_resolver); - - if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) { - op_resolver_ptr = &(cc->InputSidePackets() - .Tag("CUSTOM_OP_RESOLVER") - .Get()); - } - -#if defined(MEDIAPIPE_EDGE_TPU) - interpreter_ = - BuildEdgeTpuInterpreter(model, op_resolver_ptr, edgetpu_context_.get()); -#else - tflite::InterpreterBuilder(model, *op_resolver_ptr)(&interpreter_); -#endif // MEDIAPIPE_EDGE_TPU - - RET_CHECK(interpreter_); - -#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU) - interpreter_->SetNumThreads(1); -#else - interpreter_->SetNumThreads( - cc->Options() - .cpu_num_thread()); -#endif // __EMSCRIPTEN__ - - if (gpu_output_) { - use_quantized_tensors_ = false; - } else { - RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk); - use_quantized_tensors_ = - (interpreter_->tensor(interpreter_->inputs()[0])->quantization.type == - kTfLiteAffineQuantization); - if (use_quantized_tensors_) gpu_inference_ = false; - } - - return absl::OkStatus(); -} - -absl::StatusOr TfLiteInferenceCalculator::GetModelAsPacket( - const CalculatorContext& cc) { - const auto& options = - cc.Options(); - if (!options.model_path().empty()) { - return TfLiteModelLoader::LoadFromPath(options.model_path()); - } - if (cc.InputSidePackets().HasTag("MODEL")) { - return cc.InputSidePackets().Tag("MODEL"); - } - return absl::Status(absl::StatusCode::kNotFound, - "Must specify TFLite model as path or loaded model."); -} - -absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) { - const auto& calculator_opts = - cc->Options(); - if (calculator_opts.has_delegate() && - calculator_opts.delegate().has_tflite()) { - // Default tflite inference requeqsted - no need to modify graph. - return absl::OkStatus(); - } - - if (!gpu_inference_) { -#if defined(MEDIAPIPE_ANDROID) - const bool nnapi_requested = calculator_opts.has_delegate() - ? calculator_opts.delegate().has_nnapi() - : calculator_opts.use_nnapi(); - if (nnapi_requested) { - // Attempt to use NNAPI. - // If not supported, the default CPU delegate will be created and used. - interpreter_->SetAllowFp16PrecisionForFp32(1); - delegate_ = - TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) { - // No need to free according to tflite::NnApiDelegate() - // documentation. - }); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - return absl::OkStatus(); - } -#endif // MEDIAPIPE_ANDROID - -#if defined(__EMSCRIPTEN__) - const bool use_xnnpack = true; -#else - const bool use_xnnpack = calculator_opts.has_delegate() && - calculator_opts.delegate().has_xnnpack(); -#endif // defined(__EMSCRIPTEN__) - -#if !defined(MEDIAPIPE_EDGE_TPU) - if (use_xnnpack) { - TfLiteXNNPackDelegateOptions xnnpack_opts{}; - xnnpack_opts.num_threads = GetXnnpackNumThreads(calculator_opts); - delegate_ = TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&xnnpack_opts), - &TfLiteXNNPackDelegateDelete); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - return absl::OkStatus(); - } -#endif // !EDGETPU - - // Return and use default tflite infernece (on CPU). No need for GPU - // delegate below. - return absl::OkStatus(); - } - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - // Configure and create the delegate. - TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault(); - options.compile_options.precision_loss_allowed = - allow_precision_loss_ ? 1 : 0; - options.compile_options.preferred_gl_object_type = - TFLITE_GL_OBJECT_TYPE_FASTEST; - options.compile_options.dynamic_batch_enabled = 0; - options.compile_options.inline_parameters = 1; - if (!delegate_) - delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options), - &TfLiteGpuDelegateDelete); - - if (gpu_input_) { - // Get input image sizes. - const auto& input_indices = interpreter_->inputs(); - gpu_data_in_.resize(input_indices.size()); - for (int i = 0; i < input_indices.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]); - gpu_data_in_[i] = absl::make_unique(); - gpu_data_in_[i]->elements = 1; - for (int d = 0; d < tensor->dims->size; ++d) { - gpu_data_in_[i]->elements *= tensor->dims->data[d]; - } - // Create and bind input buffer. - MP_RETURN_IF_ERROR( - ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer( - gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer)); - RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( - delegate_.get(), gpu_data_in_[i]->buffer.id(), - interpreter_->inputs()[i]), - kTfLiteOk); - } - } - if (gpu_output_) { - // Get output image sizes. - const auto& output_indices = interpreter_->outputs(); - gpu_data_out_.resize(output_indices.size()); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]); - gpu_data_out_[i] = absl::make_unique(); - gpu_data_out_[i]->elements = 1; - // TODO handle *2 properly on some dialated models - for (int d = 0; d < tensor->dims->size; ++d) { - gpu_data_out_[i]->elements *= tensor->dims->data[d]; - } - } - // Create and bind output buffers. - interpreter_->SetAllowBufferHandleOutput(true); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer)); - RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( - delegate_.get(), gpu_data_out_[i]->buffer.id(), - output_indices[i]), - kTfLiteOk); - } - } - - // Must call this last. - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE - const int kHalfSize = 2; // sizeof(half) - // Configure and create the delegate. - TFLGpuDelegateOptions options; - // `enable_quantization` enables the run of sparse models i.e. the models with - // DENSIFY op preceding DEQUINTIZE op. Both ops get removed from the execution - // graph after the tensor of the weights is read. - options.enable_quantization = true; - options.allow_precision_loss = allow_precision_loss_; - options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypeActive; - if (!delegate_) - delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), - &TFLGpuDelegateDelete); - id device = gpu_helper_.mtlDevice; - - if (gpu_input_) { - // Get input image sizes. - const auto& input_indices = interpreter_->inputs(); - gpu_data_in_.resize(input_indices.size()); - for (int i = 0; i < input_indices.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]); - gpu_data_in_[i] = absl::make_unique(); - gpu_data_in_[i]->shape.b = tensor->dims->data[0]; - gpu_data_in_[i]->shape.h = tensor->dims->data[1]; - gpu_data_in_[i]->shape.w = tensor->dims->data[2]; - // On iOS GPU, input must be 4 channels, regardless of what model expects. - gpu_data_in_[i]->shape.c = 4; - gpu_data_in_[i]->elements = - gpu_data_in_[i]->shape.b * gpu_data_in_[i]->shape.h * - gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c; - // Input to model can be RGBA only. - if (tensor->dims->data[3] != 4) { - LOG(WARNING) << "Please ensure input GPU tensor is 4 channels."; - } - const std::string shader_source = - absl::Substitute(R"(#include - using namespace metal; - kernel void convertKernel(device float4* const input_buffer [[buffer(0)]], - device half4* output_buffer [[buffer(1)]], - uint gid [[thread_position_in_grid]]) { - if (gid >= $0) return; - output_buffer[gid] = half4(input_buffer[gid]); - })", - gpu_data_in_[i]->elements / 4); - NSString* library_source = - [NSString stringWithUTF8String:shader_source.c_str()]; - NSError* error = nil; - id library = - [device newLibraryWithSource:library_source options:nil error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"convertKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - fp32_to_fp16_program_ = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(fp32_to_fp16_program_ != nil) - << "Couldn't create pipeline state " - << [[error localizedDescription] UTF8String]; - - // Create and bind input buffer. - gpu_data_in_[i]->buffer = - [device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize - options:MTLResourceStorageModeShared]; - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), - kTfLiteOk); - RET_CHECK_EQ( - TFLGpuDelegateBindMetalBufferToTensor( - delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer), - true); - } - } - if (gpu_output_) { - // Get output image sizes. - const auto& output_indices = interpreter_->outputs(); - gpu_data_out_.resize(output_indices.size()); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]); - gpu_data_out_[i] = absl::make_unique(); - gpu_data_out_[i]->elements = 1; - // TODO handle *2 properly on some dialated models - for (int d = 0; d < tensor->dims->size; ++d) { - // Pad each dim for BHWC4 conversion inside delegate. - gpu_data_out_[i]->elements *= RoundUp(tensor->dims->data[d], 4); - } - // Save dimensions for reshaping back later. - gpu_data_out_[i]->shape.b = tensor->dims->data[0]; - switch (tensor->dims->size) { - case 2: - gpu_data_out_[i]->shape.h = 1; - gpu_data_out_[i]->shape.w = 1; - gpu_data_out_[i]->shape.c = tensor->dims->data[1]; - break; - case 3: - gpu_data_out_[i]->shape.h = 1; - gpu_data_out_[i]->shape.w = tensor->dims->data[1]; - gpu_data_out_[i]->shape.c = tensor->dims->data[2]; - break; - case 4: - gpu_data_out_[i]->shape.h = tensor->dims->data[1]; - gpu_data_out_[i]->shape.w = tensor->dims->data[2]; - gpu_data_out_[i]->shape.c = tensor->dims->data[3]; - break; - default: - return absl::InternalError("Unsupported tensor shape."); - } - } - // Create and bind output buffers. - interpreter_->SetAllowBufferHandleOutput(true); - for (int i = 0; i < gpu_data_out_.size(); ++i) { - gpu_data_out_[i]->buffer = - [device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize - options:MTLResourceStorageModeShared]; - RET_CHECK_EQ( - TFLGpuDelegateBindMetalBufferToTensor( - delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer), - true); - } - - // Create converter for GPU output. - converter_from_BPHWC4_ = - [[TFLBufferConvert alloc] initWithDevice:device - isFloat16:allow_precision_loss_ - convertToPBHWC4:false]; - if (converter_from_BPHWC4_ == nil) { - return absl::InternalError( - "Error initializating output buffer converter"); - } - } -#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.proto b/mediapipe/calculators/tflite/tflite_inference_calculator.proto deleted file mode 100644 index 02dc20831..000000000 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.proto +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Full Example: -// -// node { -// calculator: "TfLiteInferenceCalculator" -// input_stream: "TENSOR_IN:image_tensors" -// output_stream: "TENSOR_OUT:result_tensors" -// options { -// [mediapipe.TfLiteInferenceCalculatorOptions.ext] { -// model_path: "model.tflite" -// delegate { gpu {} } -// } -// } -// } -// -message TfLiteInferenceCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TfLiteInferenceCalculatorOptions ext = 233867213; - } - - message Delegate { - // Default inference provided by tflite. - message TfLite {} - // Delegate to run GPU inference depending on the device. - // (Can use OpenGl, OpenCl, Metal depending on the device.) - message Gpu { - // Experimental, Android/Linux only. Use TFLite GPU delegate API2 for - // the NN inference. - // example: - // delegate: { gpu { use_advanced_gpu_api: true } } - optional bool use_advanced_gpu_api = 1 [default = false]; - - // This option is valid for TFLite GPU delegate API2 only, - // Choose any of available APIs to force running inference using it. - enum Api { - ANY = 0; - OPENGL = 1; - OPENCL = 2; - } - optional Api api = 4 [default = ANY]; - - // This option is valid for TFLite GPU delegate API2 only, - // Set to true to use 16-bit float precision. If max precision is needed, - // set to false for 32-bit float calculations only. - optional bool allow_precision_loss = 3 [default = true]; - - // Load pre-compiled serialized binary cache to accelerate init process. - // Only available for OpenCL delegate on Android. - // Kernel caching will only be enabled if this path is set. - optional string cached_kernel_path = 2; - } - // Android only. - message Nnapi {} - message Xnnpack { - // Number of threads for XNNPACK delegate. (By default, calculator tries - // to choose optimal number of threads depending on the device.) - optional int32 num_threads = 1 [default = -1]; - } - - oneof delegate { - TfLite tflite = 1; - Gpu gpu = 2; - Nnapi nnapi = 3; - Xnnpack xnnpack = 4; - } - } - - // Path to the TF Lite model (ex: /path/to/modelname.tflite). - // On mobile, this is generally just modelname.tflite. - optional string model_path = 1; - - // Whether the TF Lite GPU or CPU backend should be used. Effective only when - // input tensors are on CPU. For input tensors on GPU, GPU backend is always - // used. - // DEPRECATED: configure "delegate" instead. - optional bool use_gpu = 2 [deprecated = true, default = false]; - - // Android only. When true, an NNAPI delegate will be used for inference. - // If NNAPI is not available, then the default CPU delegate will be used - // automatically. - // DEPRECATED: configure "delegate" instead. - optional bool use_nnapi = 3 [deprecated = true, default = false]; - - // The number of threads available to the interpreter. Effective only when - // input tensors are on CPU and 'use_gpu' is false. - optional int32 cpu_num_thread = 4 [default = -1]; - - // TfLite delegate to run inference. - // If not specified, when any of the input and output is on GPU (i.e, using - // the TENSORS_GPU tag) TFLite GPU delegate is used (as if "gpu {}" is - // specified), or otherwise regular TFLite on CPU is used (as if "tflite {}" - // is specified) except when building with emscripten where xnnpack is used. - // NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes - // precedence over use_* deprecated options.) - optional Delegate delegate = 5; -} diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc b/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc deleted file mode 100644 index ec16d1842..000000000 --- a/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tflite/tflite_inference_calculator_test_common.h" - -namespace mediapipe { - -// Tests a simple add model that adds an input tensor to itself. -TEST(TfLiteInferenceCalculatorTest, SmokeTest) { - std::string graph_proto = R"( - input_stream: "tensor_in" - node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - options { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/calculators/tflite/testdata/add.bin" - $delegate - } - } - } - )"; - // Test CPU inference only. - DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll( - graph_proto, {{"$delegate", "delegate { tflite {} }"}})); - DoSmokeTest(absl::StrReplaceAll( - graph_proto, {{"$delegate", "delegate { xnnpack {} }"}})); - DoSmokeTest(absl::StrReplaceAll( - graph_proto, - {{"$delegate", "delegate { xnnpack { num_threads: 10 } }"}})); -} - -TEST(TfLiteInferenceCalculatorTest, SmokeTest_ModelAsInputSidePacket) { - std::string graph_proto = R"( - input_stream: "tensor_in" - - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:model_path" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { string_value: "mediapipe/calculators/tflite/testdata/add.bin" } - } - } - } - - node { - calculator: "LocalFileContentsCalculator" - input_side_packet: "FILE_PATH:model_path" - output_side_packet: "CONTENTS:model_blob" - } - - node { - calculator: "TfLiteModelCalculator" - input_side_packet: "MODEL_BLOB:model_blob" - output_side_packet: "MODEL:model" - } - - node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - input_side_packet: "MODEL:model" - options { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - use_gpu: false - delegate { tflite {} } - } - } - } - )"; - DoSmokeTest(graph_proto); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator_test_common.h b/mediapipe/calculators/tflite/tflite_inference_calculator_test_common.h deleted file mode 100644 index cf995f47b..000000000 --- a/mediapipe/calculators/tflite/tflite_inference_calculator_test_common.h +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TFLITE_TFLITE_INFERENCE_CALCULATOR_TEST_H_ -#define MEDIAPIPE_CALCULATORS_TFLITE_TFLITE_INFERENCE_CALCULATOR_TEST_H_ - -#include -#include -#include - -#include "absl/strings/str_replace.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "mediapipe/framework/tool/validate_type.h" -#include "tensorflow/lite/error_reporter.h" -#include "tensorflow/lite/interpreter.h" -#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" -#include "tensorflow/lite/portable_type_to_tflitetype.h" - -#ifdef __APPLE__ -#include -#endif // defined(__APPLE__) - -namespace mediapipe { - -using ::tflite::Interpreter; - -template -void DoSmokeTest(const std::string& graph_proto) { - const int width = 8; - const int height = 8; - const int channels = 3; - - static_assert(std::is_same_v || std::is_same_v, - "Only float & uint8 currently supported."); - - // Prepare interpreter and input tensor. - std::unique_ptr interpreter(new Interpreter); - ASSERT_NE(interpreter, nullptr); - - interpreter->AddTensors(1); - interpreter->SetInputs({0}); - interpreter->SetOutputs({0}); - TfLiteQuantization quant; - if (std::is_integral_v) { - auto* affine_quant = static_cast( - malloc(sizeof(TfLiteAffineQuantization))); - affine_quant->scale = TfLiteFloatArrayCreate(1); - affine_quant->zero_point = TfLiteIntArrayCreate(1); - affine_quant->scale->data[0] = 1.0; - affine_quant->zero_point->data[0] = 0; - quant.type = kTfLiteAffineQuantization; - quant.params = affine_quant; - } - interpreter->SetTensorParametersReadWrite(0, tflite::typeToTfLiteType(), - "", {3}, quant); - - int t = interpreter->inputs()[0]; - TfLiteTensor* input_tensor = interpreter->tensor(t); - interpreter->ResizeInputTensor(t, {width, height, channels}); - interpreter->AllocateTensors(); - - T* input_tensor_buffer = tflite::GetTensorData(input_tensor); - ASSERT_NE(input_tensor_buffer, nullptr); - for (int i = 0; i < width * height * channels - 1; i++) { - input_tensor_buffer[i] = 1; - } - - auto input_vec = absl::make_unique>(); - input_vec->emplace_back(*input_tensor); - - // Prepare single calculator graph to and wait for packets. - CalculatorGraphConfig graph_config = - ParseTextProtoOrDie(graph_proto); - std::vector output_packets; - tool::AddVectorSink("tensor_out", &graph_config, &output_packets); - CalculatorGraph graph(graph_config); - MP_ASSERT_OK(graph.StartRun({})); - - // Push the tensor into the graph. - MP_ASSERT_OK(graph.AddPacketToInputStream( - "tensor_in", Adopt(input_vec.release()).At(Timestamp(0)))); - // Wait until the calculator done processing. - MP_ASSERT_OK(graph.WaitUntilIdle()); - ASSERT_EQ(1, output_packets.size()); - - // Get and process results. - const std::vector& result_vec = - output_packets[0].Get>(); - ASSERT_EQ(1, result_vec.size()); - - const TfLiteTensor* result = &result_vec[0]; - const T* result_buffer = tflite::GetTensorData(result); - ASSERT_NE(result_buffer, nullptr); - for (int i = 0; i < width * height * channels - 1; i++) { - ASSERT_EQ(3, result_buffer[i]); - } - - // Fully close graph at end, otherwise calculator+tensors are destroyed - // after calling WaitUntilDone(). - MP_ASSERT_OK(graph.CloseInputStream("tensor_in")); - MP_ASSERT_OK(graph.WaitUntilDone()); -} - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TFLITE_TFLITE_INFERENCE_CALCULATOR_TEST_H_ diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator_tpu_test.cc b/mediapipe/calculators/tflite/tflite_inference_calculator_tpu_test.cc deleted file mode 100644 index eac0d361c..000000000 --- a/mediapipe/calculators/tflite/tflite_inference_calculator_tpu_test.cc +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/str_replace.h" -#include "mediapipe/calculators/tflite/tflite_inference_calculator_test_common.h" - -namespace mediapipe { - -// Tests a simple add model that adds an input tensor to itself. -TEST(TfLiteInferenceCalculatorTpuTest, SmokeTest) { - std::string graph_proto = R"( - input_stream: "tensor_in" - node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - options { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/calculators/tflite/testdata/add_quantized.bin" - $delegate - } - } - } - )"; - DoSmokeTest( - /*graph_proto=*/absl::StrReplaceAll(graph_proto, {{"$delegate", ""}})); - DoSmokeTest(/*graph_proto=*/absl::StrReplaceAll( - graph_proto, {{"$delegate", "delegate { tflite {} }"}})); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_model_calculator.cc b/mediapipe/calculators/tflite/tflite_model_calculator.cc deleted file mode 100644 index ca28910e5..000000000 --- a/mediapipe/calculators/tflite/tflite_model_calculator.cc +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/ret_check.h" -#include "tensorflow/lite/model.h" - -namespace mediapipe { - -// Loads TfLite model from model blob specified as input side packet and outputs -// corresponding side packet. -// -// Input side packets: -// MODEL_BLOB - TfLite model blob/file-contents (std::string). You can read -// model blob from file (using whatever APIs you have) and pass -// it to the graph as input side packet or you can use some of -// calculators like LocalFileContentsCalculator to get model -// blob and use it as input here. -// -// Output side packets: -// MODEL - TfLite model. (std::unique_ptr>) -// -// Example use: -// -// node { -// calculator: "TfLiteModelCalculator" -// input_side_packet: "MODEL_BLOB:model_blob" -// output_side_packet: "MODEL:model" -// } -// -class TfLiteModelCalculator : public CalculatorBase { - public: - using TfLiteModelPtr = - std::unique_ptr>; - - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag("MODEL_BLOB").Set(); - cc->OutputSidePackets().Tag("MODEL").Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const Packet& model_packet = cc->InputSidePackets().Tag("MODEL_BLOB"); - const std::string& model_blob = model_packet.Get(); - std::unique_ptr model = - tflite::FlatBufferModel::BuildFromBuffer(model_blob.data(), - model_blob.size()); - RET_CHECK(model) << "Failed to load TfLite model from blob."; - - cc->OutputSidePackets().Tag("MODEL").Set( - MakePacket(TfLiteModelPtr( - model.release(), [model_packet](tflite::FlatBufferModel* model) { - // Keeping model_packet in order to keep underlying model blob - // which can be released only after TfLite model is not needed - // anymore (deleted). - delete model; - }))); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(TfLiteModelCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_model_calculator_test.cc b/mediapipe/calculators/tflite/tflite_model_calculator_test.cc deleted file mode 100644 index 6b4b18c2d..000000000 --- a/mediapipe/calculators/tflite/tflite_model_calculator_test.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT -#include "tensorflow/lite/model.h" - -namespace mediapipe { - -TEST(TfLiteModelCalculatorTest, SmokeTest) { - // Prepare single calculator graph to and wait for packets. - CalculatorGraphConfig graph_config = ParseTextProtoOrDie< - CalculatorGraphConfig>( - R"pb( - node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:model_path" - options: { - [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { - packet { - string_value: "mediapipe/calculators/tflite/testdata/add.bin" - } - } - } - } - - node { - calculator: "LocalFileContentsCalculator" - input_side_packet: "FILE_PATH:model_path" - output_side_packet: "CONTENTS:model_blob" - } - - node { - calculator: "TfLiteModelCalculator" - input_side_packet: "MODEL_BLOB:model_blob" - output_side_packet: "MODEL:model" - } - )pb"); - CalculatorGraph graph(graph_config); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.WaitUntilIdle()); - auto status_or_packet = graph.GetOutputSidePacket("model"); - MP_ASSERT_OK(status_or_packet); - auto model_packet = status_or_packet.value(); - const auto& model = model_packet.Get< - std::unique_ptr>>(); - - auto expected_model = tflite::FlatBufferModel::BuildFromFile( - "mediapipe/calculators/tflite/testdata/add.bin"); - - EXPECT_EQ(model->GetModel()->version(), - expected_model->GetModel()->version()); - EXPECT_EQ(model->GetModel()->buffers()->size(), - expected_model->GetModel()->buffers()->size()); - const int num_subgraphs = expected_model->GetModel()->subgraphs()->size(); - EXPECT_EQ(model->GetModel()->subgraphs()->size(), num_subgraphs); - for (int i = 0; i < num_subgraphs; ++i) { - const auto* expected_subgraph = - expected_model->GetModel()->subgraphs()->Get(i); - const auto* subgraph = model->GetModel()->subgraphs()->Get(i); - const int num_tensors = expected_subgraph->tensors()->size(); - EXPECT_EQ(subgraph->tensors()->size(), num_tensors); - for (int j = 0; j < num_tensors; ++j) { - EXPECT_EQ(subgraph->tensors()->Get(j)->name()->str(), - expected_subgraph->tensors()->Get(j)->name()->str()); - } - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc deleted file mode 100644 index 4d28b91e9..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/container/node_hash_map.h" -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -// Convert result TFLite tensors from classification models into MediaPipe -// classifications. -// -// Input: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 containing one -// tensor, the size of which must be (1, * num_classes). -// Output: -// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index -// fields of each classification are set, while the label -// field is only set if label_map_path is provided. -// -// Usage example: -// node { -// calculator: "TfLiteTensorsToClassificationCalculator" -// input_stream: "TENSORS:tensors" -// output_stream: "CLASSIFICATIONS:classifications" -// options: { -// [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { -// num_classes: 1024 -// min_score_threshold: 0.1 -// label_map_path: "labelmap.txt" -// } -// } -// } -class TfLiteTensorsToClassificationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions options_; - int top_k_ = 0; - absl::node_hash_map label_map_; - bool label_map_loaded_ = false; -}; -REGISTER_CALCULATOR(TfLiteTensorsToClassificationCalculator); - -absl::Status TfLiteTensorsToClassificationCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - if (cc->Inputs().HasTag("TENSORS")) { - cc->Inputs().Tag("TENSORS").Set>(); - } - - if (cc->Outputs().HasTag("CLASSIFICATIONS")) { - cc->Outputs().Tag("CLASSIFICATIONS").Set(); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToClassificationCalculator::Open( - CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options< - ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions>(); - - top_k_ = options_.top_k(); - if (options_.has_label_map_path()) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, - PathToResourceAsFile(options_.label_map_path())); - std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - label_map_loaded_ = true; - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToClassificationCalculator::Process( - CalculatorContext* cc) { - const auto& input_tensors = - cc->Inputs().Tag("TENSORS").Get>(); - - RET_CHECK_EQ(input_tensors.size(), 1); - - const TfLiteTensor* raw_score_tensor = &input_tensors[0]; - int num_classes = 1; - for (int i = 0; i < raw_score_tensor->dims->size; ++i) { - num_classes *= raw_score_tensor->dims->data[i]; - } - - if (options_.binary_classification()) { - RET_CHECK_EQ(num_classes, 1); - // Number of classes for binary classification. - num_classes = 2; - } - if (label_map_loaded_) { - RET_CHECK_EQ(num_classes, label_map_.size()); - } - const float* raw_scores = raw_score_tensor->data.f; - - auto classification_list = absl::make_unique(); - if (options_.binary_classification()) { - Classification* class_first = classification_list->add_classification(); - Classification* class_second = classification_list->add_classification(); - class_first->set_index(0); - class_second->set_index(1); - class_first->set_score(raw_scores[0]); - class_second->set_score(1. - raw_scores[0]); - - if (label_map_loaded_) { - class_first->set_label(label_map_[0]); - class_second->set_label(label_map_[1]); - } - } else { - for (int i = 0; i < num_classes; ++i) { - if (options_.has_min_score_threshold() && - raw_scores[i] < options_.min_score_threshold()) { - continue; - } - Classification* classification = - classification_list->add_classification(); - classification->set_index(i); - classification->set_score(raw_scores[i]); - - if (label_map_loaded_) { - classification->set_label(label_map_[i]); - } - } - } - - // Note that partial_sort will raise error when top_k_ > - // classification_list->classification_size(). - CHECK_GE(classification_list->classification_size(), top_k_); - auto raw_classification_list = classification_list->mutable_classification(); - if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { - std::partial_sort(raw_classification_list->begin(), - raw_classification_list->begin() + top_k_, - raw_classification_list->end(), - [](const Classification a, const Classification b) { - return a.score() > b.score(); - }); - - // Resizes the underlying list to have only top_k_ classifications. - raw_classification_list->DeleteSubrange( - top_k_, raw_classification_list->size() - top_k_); - } - cc->Outputs() - .Tag("CLASSIFICATIONS") - .Add(classification_list.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToClassificationCalculator::Close( - CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto deleted file mode 100644 index c6c9d915d..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TfLiteTensorsToClassificationCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TfLiteTensorsToClassificationCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463; - } - - // Score threshold for perserving the class. - optional float min_score_threshold = 1; - // Number of highest scoring labels to output. If top_k is not positive then - // all labels are used. - optional int32 top_k = 2; - // Path to a label map file for getting the actual name of class ids. - optional string label_map_path = 3; - // Whether the input is a single float for binary classification. - // When true, only a single float is expected in the input tensor and the - // label map, if provided, is expected to have exactly two labels. - // The single score(float) represent the probability of first label, and - // 1 - score is the probabilility of the second label. - optional bool binary_classification = 4; -} diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc deleted file mode 100644 index 37545b8a8..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { - -using mediapipe::ParseTextProtoOrDie; -using ::tflite::Interpreter; -using Node = ::mediapipe::CalculatorGraphConfig::Node; - -class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test { - protected: - void BuildGraph(mediapipe::CalculatorRunner* runner, - const std::vector& scores) { - interpreter_ = absl::make_unique(); - - std::vector dims(2); - dims[0] = 1; - dims[1] = scores.size(); - - interpreter_->AddTensors(1); - interpreter_->SetInputs({0}); - interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", dims, - TfLiteQuantization()); - - int t = interpreter_->inputs()[0]; - TfLiteTensor* tensor = interpreter_->tensor(t); - interpreter_->ResizeInputTensor(t, dims); - interpreter_->AllocateTensors(); - - float* tensor_buffer = tensor->data.f; - ASSERT_NE(tensor_buffer, nullptr); - for (int i = 0; i < scores.size(); ++i) { - tensor_buffer[i] = scores[i]; - } - - auto tensors = absl::make_unique>(); - tensors->emplace_back(*tensor); - - int64 stream_timestamp = 0; - auto& input_stream_packets = - runner->MutableInputs()->Tag("TENSORS").packets; - - input_stream_packets.push_back( - mediapipe::Adopt(tensors.release()) - .At(mediapipe::Timestamp(stream_timestamp++))); - } - - std::unique_ptr interpreter_; -}; - -TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TfLiteTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {} - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label_id and score fields are set correctly. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_FALSE(classification_list.classification(i).has_label()); - } -} - -TEST_F(TfLiteTensorsToClassificationCalculatorTest, - CorrectOutputWithLabelMapPath) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TfLiteTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { - label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt" - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - EXPECT_EQ(3, classification_list.classification_size()); - - // Verify that the label field is set. - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ(i, classification_list.classification(i).index()); - EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); - ASSERT_TRUE(classification_list.classification(i).has_label()); - } -} - -TEST_F(TfLiteTensorsToClassificationCalculatorTest, - CorrectOutputWithLabelMinScoreThreshold) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TfLiteTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { - min_score_threshold: 0.6 - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the low score labels are filtered out. - EXPECT_EQ(1, classification_list.classification_size()); - EXPECT_EQ(1, classification_list.classification(0).score()); -} - -TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) { - mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TfLiteTensorsToClassificationCalculator" - input_stream: "TENSORS:tensors" - output_stream: "CLASSIFICATIONS:classifications" - options { - [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { - top_k: 2 - } - } - )pb")); - - BuildGraph(&runner, {0, 0.5, 1}); - MP_ASSERT_OK(runner.Run()); - - const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; - - EXPECT_EQ(1, output_packets_.size()); - - const auto& classification_list = - output_packets_[0].Get(); - - // Verify that the only top2 labels are left. - EXPECT_EQ(2, classification_list.classification_size()); - for (int i = 0; i < classification_list.classification_size(); ++i) { - EXPECT_EQ((classification_list.classification_size() - i) * 0.5, - classification_list.classification(i).score()); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc deleted file mode 100644 index 2ed62c46d..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc +++ /dev/null @@ -1,1158 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/formats/object_detection/anchor.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/tflite/config.h" -#include "tensorflow/lite/interpreter.h" - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_program.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -#if MEDIAPIPE_TFLITE_METAL_INFERENCE -#import -#import -#import - -#import "mediapipe/gpu/MPPMetalHelper.h" -#include "mediapipe/gpu/MPPMetalUtil.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "tensorflow/lite/delegates/gpu/metal_delegate.h" -#endif // MEDIAPIPE_TFLITE_METAL_INFERENCE - -namespace { -constexpr int kNumInputTensorsWithAnchors = 3; -constexpr int kNumCoordsPerBox = 4; - -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kTensorsGpuTag[] = "TENSORS_GPU"; -} // namespace - -namespace mediapipe { - -#if MEDIAPIPE_TFLITE_GL_INFERENCE -using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer; -using ::tflite::gpu::gl::GlShader; -typedef ::tflite::gpu::gl::GlProgram GpuProgram; -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE -typedef id GpuProgram; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - -namespace { - -#if MEDIAPIPE_TFLITE_GPU_SUPPORTED -struct GPUData { - GpuProgram decode_program; - GpuProgram score_program; - GpuTensor decoded_boxes_buffer; - GpuTensor raw_boxes_buffer; - GpuTensor raw_anchors_buffer; - GpuTensor scored_boxes_buffer; - GpuTensor raw_scores_buffer; -}; -#endif // MEDIAPIPE_TFLITE_GPU_SUPPORTED - -void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes, - std::vector* anchors) { - anchors->clear(); - for (int i = 0; i < num_boxes; ++i) { - Anchor new_anchor; - new_anchor.set_y_center(raw_anchors[i * kNumCoordsPerBox + 0]); - new_anchor.set_x_center(raw_anchors[i * kNumCoordsPerBox + 1]); - new_anchor.set_h(raw_anchors[i * kNumCoordsPerBox + 2]); - new_anchor.set_w(raw_anchors[i * kNumCoordsPerBox + 3]); - anchors->push_back(new_anchor); - } -} - -void ConvertAnchorsToRawValues(const std::vector& anchors, - int num_boxes, float* raw_anchors) { - CHECK_EQ(anchors.size(), num_boxes); - int box = 0; - for (const auto& anchor : anchors) { - raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); - raw_anchors[box * kNumCoordsPerBox + 1] = anchor.x_center(); - raw_anchors[box * kNumCoordsPerBox + 2] = anchor.h(); - raw_anchors[box * kNumCoordsPerBox + 3] = anchor.w(); - ++box; - } -} - -} // namespace - -// Convert result TFLite tensors from object detection models into MediaPipe -// Detections. -// -// Input: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. The vector of -// tensors can have 2 or 3 tensors. First tensor is the predicted -// raw boxes/keypoints. The size of the values must be (num_boxes -// * num_predicted_values). Second tensor is the score tensor. The -// size of the valuse must be (num_boxes * num_classes). It's -// optional to pass in a third tensor for anchors (e.g. for SSD -// models) depend on the outputs of the detection model. The size -// of anchor tensor must be (num_boxes * 4). -// TENSORS_GPU - vector of GlBuffer of MTLBuffer. -// Output: -// DETECTIONS - Result MediaPipe detections. -// -// Usage example: -// node { -// calculator: "TfLiteTensorsToDetectionsCalculator" -// input_stream: "TENSORS:tensors" -// input_side_packet: "ANCHORS:anchors" -// output_stream: "DETECTIONS:detections" -// options: { -// [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { -// num_classes: 91 -// num_boxes: 1917 -// num_coords: 4 -// ignore_classes: [0, 1, 2] -// x_scale: 10.0 -// y_scale: 10.0 -// h_scale: 5.0 -// w_scale: 5.0 -// } -// } -// } -class TfLiteTensorsToDetectionsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status ProcessCPU(CalculatorContext* cc, - std::vector* output_detections); - absl::Status ProcessGPU(CalculatorContext* cc, - std::vector* output_detections); - - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status GpuInit(CalculatorContext* cc); - absl::Status DecodeBoxes(const float* raw_boxes, - const std::vector& anchors, - std::vector* boxes); - absl::Status ConvertToDetections(const float* detection_boxes, - const float* detection_scores, - const int* detection_classes, - std::vector* output_detections); - Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, - float box_xmax, float score, int class_id, - bool flip_vertically); - - int num_classes_ = 0; - int num_boxes_ = 0; - int num_coords_ = 0; - std::set ignore_classes_; - - ::mediapipe::TfLiteTensorsToDetectionsCalculatorOptions options_; - std::vector anchors_; - bool side_packet_anchors_{}; - -#if MEDIAPIPE_TFLITE_GL_INFERENCE - mediapipe::GlCalculatorHelper gpu_helper_; - std::unique_ptr gpu_data_; -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MPPMetalHelper* gpu_helper_ = nullptr; - std::unique_ptr gpu_data_; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - bool gpu_input_ = false; - bool anchors_init_ = false; -}; -REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator); - -absl::Status TfLiteTensorsToDetectionsCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - bool use_gpu = false; - - if (cc->Inputs().HasTag(kTensorsTag)) { - cc->Inputs().Tag(kTensorsTag).Set>(); - } - - if (cc->Inputs().HasTag(kTensorsGpuTag)) { - cc->Inputs().Tag(kTensorsGpuTag).Set>(); - use_gpu |= true; - } - - if (cc->Outputs().HasTag("DETECTIONS")) { - cc->Outputs().Tag("DETECTIONS").Set>(); - } - - if (cc->InputSidePackets().UsesTags()) { - if (cc->InputSidePackets().HasTag("ANCHORS")) { - cc->InputSidePackets().Tag("ANCHORS").Set>(); - } - } - - if (use_gpu) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kTensorsGpuTag)) { - gpu_input_ = true; -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - } - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - side_packet_anchors_ = cc->InputSidePackets().HasTag("ANCHORS"); - - if (gpu_input_) { - MP_RETURN_IF_ERROR(GpuInit(cc)); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::Process( - CalculatorContext* cc) { - if ((!gpu_input_ && cc->Inputs().Tag(kTensorsTag).IsEmpty()) || - (gpu_input_ && cc->Inputs().Tag(kTensorsGpuTag).IsEmpty())) { - return absl::OkStatus(); - } - - auto output_detections = absl::make_unique>(); - - if (gpu_input_) { - MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get())); - } else { - MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get())); - } - - // Output - if (cc->Outputs().HasTag("DETECTIONS")) { - cc->Outputs() - .Tag("DETECTIONS") - .Add(output_detections.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU( - CalculatorContext* cc, std::vector* output_detections) { - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - - if (input_tensors.size() == 2 || - input_tensors.size() == kNumInputTensorsWithAnchors) { - // Postprocessing on CPU for model without postprocessing op. E.g. output - // raw score tensor and box tensor. Anchor decoding will be handled below. - const TfLiteTensor* raw_box_tensor = &input_tensors[0]; - const TfLiteTensor* raw_score_tensor = &input_tensors[1]; - - // TODO: Add flexible input tensor size handling. - CHECK_EQ(raw_box_tensor->dims->size, 3); - CHECK_EQ(raw_box_tensor->dims->data[0], 1); - CHECK_EQ(raw_box_tensor->dims->data[1], num_boxes_); - CHECK_EQ(raw_box_tensor->dims->data[2], num_coords_); - CHECK_EQ(raw_score_tensor->dims->size, 3); - CHECK_EQ(raw_score_tensor->dims->data[0], 1); - CHECK_EQ(raw_score_tensor->dims->data[1], num_boxes_); - CHECK_EQ(raw_score_tensor->dims->data[2], num_classes_); - const float* raw_boxes = raw_box_tensor->data.f; - const float* raw_scores = raw_score_tensor->data.f; - - // TODO: Support other options to load anchors. - if (!anchors_init_) { - if (input_tensors.size() == kNumInputTensorsWithAnchors) { - const TfLiteTensor* anchor_tensor = &input_tensors[2]; - CHECK_EQ(anchor_tensor->dims->size, 2); - CHECK_EQ(anchor_tensor->dims->data[0], num_boxes_); - CHECK_EQ(anchor_tensor->dims->data[1], kNumCoordsPerBox); - const float* raw_anchors = anchor_tensor->data.f; - ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_); - } else if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); - anchors_ = - cc->InputSidePackets().Tag("ANCHORS").Get>(); - } else { - return absl::UnavailableError("No anchor data available."); - } - anchors_init_ = true; - } - std::vector boxes(num_boxes_ * num_coords_); - MP_RETURN_IF_ERROR(DecodeBoxes(raw_boxes, anchors_, &boxes)); - - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - - // Filter classes by scores. - for (int i = 0; i < num_boxes_; ++i) { - int class_id = -1; - float max_score = -std::numeric_limits::max(); - // Find the top score for box i. - for (int score_idx = 0; score_idx < num_classes_; ++score_idx) { - if (ignore_classes_.find(score_idx) == ignore_classes_.end()) { - auto score = raw_scores[i * num_classes_ + score_idx]; - if (options_.sigmoid_score()) { - if (options_.has_score_clipping_thresh()) { - score = score < -options_.score_clipping_thresh() - ? -options_.score_clipping_thresh() - : score; - score = score > options_.score_clipping_thresh() - ? options_.score_clipping_thresh() - : score; - } - score = 1.0f / (1.0f + std::exp(-score)); - } - if (max_score < score) { - max_score = score; - class_id = score_idx; - } - } - } - detection_scores[i] = max_score; - detection_classes[i] = class_id; - } - - MP_RETURN_IF_ERROR( - ConvertToDetections(boxes.data(), detection_scores.data(), - detection_classes.data(), output_detections)); - } else { - // Postprocessing on CPU with postprocessing op (e.g. anchor decoding and - // non-maximum suppression) within the model. - RET_CHECK_EQ(input_tensors.size(), 4); - - const TfLiteTensor* detection_boxes_tensor = &input_tensors[0]; - const TfLiteTensor* detection_classes_tensor = &input_tensors[1]; - const TfLiteTensor* detection_scores_tensor = &input_tensors[2]; - const TfLiteTensor* num_boxes_tensor = &input_tensors[3]; - RET_CHECK_EQ(num_boxes_tensor->dims->size, 1); - RET_CHECK_EQ(num_boxes_tensor->dims->data[0], 1); - const float* num_boxes = num_boxes_tensor->data.f; - num_boxes_ = num_boxes[0]; - RET_CHECK_EQ(detection_boxes_tensor->dims->size, 3); - RET_CHECK_EQ(detection_boxes_tensor->dims->data[0], 1); - const int max_detections = detection_boxes_tensor->dims->data[1]; - RET_CHECK_EQ(detection_boxes_tensor->dims->data[2], num_coords_); - RET_CHECK_EQ(detection_classes_tensor->dims->size, 2); - RET_CHECK_EQ(detection_classes_tensor->dims->data[0], 1); - RET_CHECK_EQ(detection_classes_tensor->dims->data[1], max_detections); - RET_CHECK_EQ(detection_scores_tensor->dims->size, 2); - RET_CHECK_EQ(detection_scores_tensor->dims->data[0], 1); - RET_CHECK_EQ(detection_scores_tensor->dims->data[1], max_detections); - - const float* detection_boxes = detection_boxes_tensor->data.f; - const float* detection_scores = detection_scores_tensor->data.f; - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { - detection_classes[i] = - static_cast(detection_classes_tensor->data.f[i]); - } - MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores, - detection_classes.data(), - output_detections)); - } - return absl::OkStatus(); -} -absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU( - CalculatorContext* cc, std::vector* output_detections) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - RET_CHECK_GE(input_tensors.size(), 2); - - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &input_tensors, &cc, - &output_detections]() - -> absl::Status { - // Copy inputs. - MP_RETURN_IF_ERROR( - CopyBuffer(input_tensors[0], gpu_data_->raw_boxes_buffer)); - MP_RETURN_IF_ERROR( - CopyBuffer(input_tensors[1], gpu_data_->raw_scores_buffer)); - if (!anchors_init_) { - if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); - const auto& anchors = - cc->InputSidePackets().Tag("ANCHORS").Get>(); - std::vector raw_anchors(num_boxes_ * kNumCoordsPerBox); - ConvertAnchorsToRawValues(anchors, num_boxes_, raw_anchors.data()); - MP_RETURN_IF_ERROR(gpu_data_->raw_anchors_buffer.Write( - absl::MakeSpan(raw_anchors))); - } else { - CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); - MP_RETURN_IF_ERROR( - CopyBuffer(input_tensors[2], gpu_data_->raw_anchors_buffer)); - } - anchors_init_ = true; - } - - // Run shaders. - // Decode boxes. - MP_RETURN_IF_ERROR(gpu_data_->decoded_boxes_buffer.BindToIndex(0)); - MP_RETURN_IF_ERROR(gpu_data_->raw_boxes_buffer.BindToIndex(1)); - MP_RETURN_IF_ERROR(gpu_data_->raw_anchors_buffer.BindToIndex(2)); - const tflite::gpu::uint3 decode_workgroups = {num_boxes_, 1, 1}; - MP_RETURN_IF_ERROR(gpu_data_->decode_program.Dispatch(decode_workgroups)); - - // Score boxes. - MP_RETURN_IF_ERROR(gpu_data_->scored_boxes_buffer.BindToIndex(0)); - MP_RETURN_IF_ERROR(gpu_data_->raw_scores_buffer.BindToIndex(1)); - const tflite::gpu::uint3 score_workgroups = {num_boxes_, 1, 1}; - MP_RETURN_IF_ERROR(gpu_data_->score_program.Dispatch(score_workgroups)); - - // Copy decoded boxes from GPU to CPU. - std::vector boxes(num_boxes_ * num_coords_); - MP_RETURN_IF_ERROR( - gpu_data_->decoded_boxes_buffer.Read(absl::MakeSpan(boxes))); - std::vector score_class_id_pairs(num_boxes_ * 2); - MP_RETURN_IF_ERROR(gpu_data_->scored_boxes_buffer.Read( - absl::MakeSpan(score_class_id_pairs))); - - // TODO: b/138851969. Is it possible to output a float vector - // for score and an int vector for class so that we can avoid copying twice? - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { - detection_scores[i] = score_class_id_pairs[i * 2]; - detection_classes[i] = static_cast(score_class_id_pairs[i * 2 + 1]); - } - MP_RETURN_IF_ERROR( - ConvertToDetections(boxes.data(), detection_scores.data(), - detection_classes.data(), output_detections)); - - return absl::OkStatus(); - })); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - RET_CHECK_GE(input_tensors.size(), 2); - - // Copy inputs. - [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_boxes_buffer - from:input_tensors[0] - blocking:false - commandBuffer:[gpu_helper_ commandBuffer]]; - [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_scores_buffer - from:input_tensors[1] - blocking:false - commandBuffer:[gpu_helper_ commandBuffer]]; - if (!anchors_init_) { - if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); - const auto& anchors = - cc->InputSidePackets().Tag("ANCHORS").Get>(); - std::vector raw_anchors(num_boxes_ * kNumCoordsPerBox); - ConvertAnchorsToRawValues(anchors, num_boxes_, raw_anchors.data()); - memcpy([gpu_data_->raw_anchors_buffer contents], raw_anchors.data(), - raw_anchors.size() * sizeof(float)); - } else { - RET_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); - [MPPMetalUtil blitMetalBufferTo:gpu_data_->raw_anchors_buffer - from:input_tensors[2] - blocking:false - commandBuffer:[gpu_helper_ commandBuffer]]; - } - anchors_init_ = true; - } - - // Run shaders. - id command_buffer = [gpu_helper_ commandBuffer]; - command_buffer.label = @"TfLiteDecodeAndScoreBoxes"; - id command_encoder = - [command_buffer computeCommandEncoder]; - [command_encoder setComputePipelineState:gpu_data_->decode_program]; - [command_encoder setBuffer:gpu_data_->decoded_boxes_buffer - offset:0 - atIndex:0]; - [command_encoder setBuffer:gpu_data_->raw_boxes_buffer offset:0 atIndex:1]; - [command_encoder setBuffer:gpu_data_->raw_anchors_buffer offset:0 atIndex:2]; - MTLSize decode_threads_per_group = MTLSizeMake(1, 1, 1); - MTLSize decode_threadgroups = MTLSizeMake(num_boxes_, 1, 1); - [command_encoder dispatchThreadgroups:decode_threadgroups - threadsPerThreadgroup:decode_threads_per_group]; - - [command_encoder setComputePipelineState:gpu_data_->score_program]; - [command_encoder setBuffer:gpu_data_->scored_boxes_buffer offset:0 atIndex:0]; - [command_encoder setBuffer:gpu_data_->raw_scores_buffer offset:0 atIndex:1]; - MTLSize score_threads_per_group = MTLSizeMake(1, num_classes_, 1); - MTLSize score_threadgroups = MTLSizeMake(num_boxes_, 1, 1); - [command_encoder dispatchThreadgroups:score_threadgroups - threadsPerThreadgroup:score_threads_per_group]; - [command_encoder endEncoding]; - [MPPMetalUtil commitCommandBufferAndWait:command_buffer]; - - // Copy decoded boxes from GPU to CPU. - std::vector boxes(num_boxes_ * num_coords_); - memcpy(boxes.data(), [gpu_data_->decoded_boxes_buffer contents], - num_boxes_ * num_coords_ * sizeof(float)); - std::vector score_class_id_pairs(num_boxes_ * 2); - memcpy(score_class_id_pairs.data(), [gpu_data_->scored_boxes_buffer contents], - num_boxes_ * 2 * sizeof(float)); - - // Output detections. - // TODO Adjust shader to avoid copying shader output twice. - std::vector detection_scores(num_boxes_); - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { - detection_scores[i] = score_class_id_pairs[i * 2]; - detection_classes[i] = static_cast(score_class_id_pairs[i * 2 + 1]); - } - MP_RETURN_IF_ERROR(ConvertToDetections(boxes.data(), detection_scores.data(), - detection_classes.data(), - output_detections)); - -#else - LOG(ERROR) << "GPU input on non-Android not supported yet."; -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::Close(CalculatorContext* cc) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - gpu_helper_.RunInGlContext([this] { gpu_data_.reset(); }); -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - gpu_data_.reset(); -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::LoadOptions( - CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = - cc->Options<::mediapipe::TfLiteTensorsToDetectionsCalculatorOptions>(); - - num_classes_ = options_.num_classes(); - num_boxes_ = options_.num_boxes(); - num_coords_ = options_.num_coords(); - - // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); - - // Check if the output size is equal to the requested boxes and keypoints. - CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + - kNumCoordsPerBox, - num_coords_); - - for (int i = 0; i < options_.ignore_classes_size(); ++i) { - ignore_classes_.insert(options_.ignore_classes(i)); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::DecodeBoxes( - const float* raw_boxes, const std::vector& anchors, - std::vector* boxes) { - for (int i = 0; i < num_boxes_; ++i) { - const int box_offset = i * num_coords_ + options_.box_coord_offset(); - - float y_center = raw_boxes[box_offset]; - float x_center = raw_boxes[box_offset + 1]; - float h = raw_boxes[box_offset + 2]; - float w = raw_boxes[box_offset + 3]; - if (options_.reverse_output_order()) { - x_center = raw_boxes[box_offset]; - y_center = raw_boxes[box_offset + 1]; - w = raw_boxes[box_offset + 2]; - h = raw_boxes[box_offset + 3]; - } - - x_center = - x_center / options_.x_scale() * anchors[i].w() + anchors[i].x_center(); - y_center = - y_center / options_.y_scale() * anchors[i].h() + anchors[i].y_center(); - - if (options_.apply_exponential_on_box_size()) { - h = std::exp(h / options_.h_scale()) * anchors[i].h(); - w = std::exp(w / options_.w_scale()) * anchors[i].w(); - } else { - h = h / options_.h_scale() * anchors[i].h(); - w = w / options_.w_scale() * anchors[i].w(); - } - - const float ymin = y_center - h / 2.f; - const float xmin = x_center - w / 2.f; - const float ymax = y_center + h / 2.f; - const float xmax = x_center + w / 2.f; - - (*boxes)[i * num_coords_ + 0] = ymin; - (*boxes)[i * num_coords_ + 1] = xmin; - (*boxes)[i * num_coords_ + 2] = ymax; - (*boxes)[i * num_coords_ + 3] = xmax; - - if (options_.num_keypoints()) { - for (int k = 0; k < options_.num_keypoints(); ++k) { - const int offset = i * num_coords_ + options_.keypoint_coord_offset() + - k * options_.num_values_per_keypoint(); - - float keypoint_y = raw_boxes[offset]; - float keypoint_x = raw_boxes[offset + 1]; - if (options_.reverse_output_order()) { - keypoint_x = raw_boxes[offset]; - keypoint_y = raw_boxes[offset + 1]; - } - - (*boxes)[offset] = keypoint_x / options_.x_scale() * anchors[i].w() + - anchors[i].x_center(); - (*boxes)[offset + 1] = - keypoint_y / options_.y_scale() * anchors[i].h() + - anchors[i].y_center(); - } - } - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToDetectionsCalculator::ConvertToDetections( - const float* detection_boxes, const float* detection_scores, - const int* detection_classes, std::vector* output_detections) { - for (int i = 0; i < num_boxes_; ++i) { - if (options_.has_min_score_thresh() && - detection_scores[i] < options_.min_score_thresh()) { - continue; - } - const int box_offset = i * num_coords_; - Detection detection = ConvertToDetection( - detection_boxes[box_offset + 0], detection_boxes[box_offset + 1], - detection_boxes[box_offset + 2], detection_boxes[box_offset + 3], - detection_scores[i], detection_classes[i], options_.flip_vertically()); - const auto& bbox = detection.location_data().relative_bounding_box(); - if (bbox.width() < 0 || bbox.height() < 0) { - // Decoded detection boxes could have negative values for width/height due - // to model prediction. Filter out those boxes since some downstream - // calculators may assume non-negative values. (b/171391719) - continue; - } - - // Add keypoints. - if (options_.num_keypoints() > 0) { - auto* location_data = detection.mutable_location_data(); - for (int kp_id = 0; kp_id < options_.num_keypoints() * - options_.num_values_per_keypoint(); - kp_id += options_.num_values_per_keypoint()) { - auto keypoint = location_data->add_relative_keypoints(); - const int keypoint_index = - box_offset + options_.keypoint_coord_offset() + kp_id; - keypoint->set_x(detection_boxes[keypoint_index + 0]); - keypoint->set_y(options_.flip_vertically() - ? 1.f - detection_boxes[keypoint_index + 1] - : detection_boxes[keypoint_index + 1]); - } - } - output_detections->emplace_back(detection); - } - return absl::OkStatus(); -} - -Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection( - float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, - int class_id, bool flip_vertically) { - Detection detection; - detection.add_score(score); - detection.add_label_id(class_id); - - LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - - LocationData::RelativeBoundingBox* relative_bbox = - location_data->mutable_relative_bounding_box(); - - relative_bbox->set_xmin(box_xmin); - relative_bbox->set_ymin(flip_vertically ? 1.f - box_ymax : box_ymin); - relative_bbox->set_width(box_xmax - box_xmin); - relative_bbox->set_height(box_ymax - box_ymin); - return detection; -} - -absl::Status TfLiteTensorsToDetectionsCalculator::GpuInit( - CalculatorContext* cc) { -#if MEDIAPIPE_TFLITE_GL_INFERENCE - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> absl::Status { - gpu_data_ = absl::make_unique(); - - // A shader to decode detection boxes. - const std::string decode_src = absl::Substitute( - R"( #version 310 es - -layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in; - -layout(location = 0) uniform vec4 scale; - -layout(std430, binding = 0) writeonly buffer Output { - float data[]; -} boxes; - -layout(std430, binding = 1) readonly buffer Input0 { - float data[]; -} raw_boxes; - -layout(std430, binding = 2) readonly buffer Input1 { - float data[]; -} raw_anchors; - -uint num_coords = uint($0); -int reverse_output_order = int($1); -int apply_exponential = int($2); -int box_coord_offset = int($3); -int num_keypoints = int($4); -int keypt_coord_offset = int($5); -int num_values_per_keypt = int($6); - -void main() { - uint g_idx = gl_GlobalInvocationID.x; // box index - uint box_offset = g_idx * num_coords + uint(box_coord_offset); - uint anchor_offset = g_idx * uint(4); // check kNumCoordsPerBox - - float y_center, x_center, h, w; - - if (reverse_output_order == int(0)) { - y_center = raw_boxes.data[box_offset + uint(0)]; - x_center = raw_boxes.data[box_offset + uint(1)]; - h = raw_boxes.data[box_offset + uint(2)]; - w = raw_boxes.data[box_offset + uint(3)]; - } else { - x_center = raw_boxes.data[box_offset + uint(0)]; - y_center = raw_boxes.data[box_offset + uint(1)]; - w = raw_boxes.data[box_offset + uint(2)]; - h = raw_boxes.data[box_offset + uint(3)]; - } - - float anchor_yc = raw_anchors.data[anchor_offset + uint(0)]; - float anchor_xc = raw_anchors.data[anchor_offset + uint(1)]; - float anchor_h = raw_anchors.data[anchor_offset + uint(2)]; - float anchor_w = raw_anchors.data[anchor_offset + uint(3)]; - - x_center = x_center / scale.x * anchor_w + anchor_xc; - y_center = y_center / scale.y * anchor_h + anchor_yc; - - if (apply_exponential == int(1)) { - h = exp(h / scale.w) * anchor_h; - w = exp(w / scale.z) * anchor_w; - } else { - h = (h / scale.w) * anchor_h; - w = (w / scale.z) * anchor_w; - } - - float ymin = y_center - h / 2.0; - float xmin = x_center - w / 2.0; - float ymax = y_center + h / 2.0; - float xmax = x_center + w / 2.0; - - boxes.data[box_offset + uint(0)] = ymin; - boxes.data[box_offset + uint(1)] = xmin; - boxes.data[box_offset + uint(2)] = ymax; - boxes.data[box_offset + uint(3)] = xmax; - - if (num_keypoints > int(0)){ - for (int k = 0; k < num_keypoints; ++k) { - int kp_offset = - int(g_idx * num_coords) + keypt_coord_offset + k * num_values_per_keypt; - float kp_y, kp_x; - if (reverse_output_order == int(0)) { - kp_y = raw_boxes.data[kp_offset + int(0)]; - kp_x = raw_boxes.data[kp_offset + int(1)]; - } else { - kp_x = raw_boxes.data[kp_offset + int(0)]; - kp_y = raw_boxes.data[kp_offset + int(1)]; - } - boxes.data[kp_offset + int(0)] = kp_x / scale.x * anchor_w + anchor_xc; - boxes.data[kp_offset + int(1)] = kp_y / scale.y * anchor_h + anchor_yc; - } - } -})", - options_.num_coords(), // box xywh - options_.reverse_output_order() ? 1 : 0, - options_.apply_exponential_on_box_size() ? 1 : 0, - options_.box_coord_offset(), options_.num_keypoints(), - options_.keypoint_coord_offset(), options_.num_values_per_keypoint()); - - // Shader program - GlShader decode_shader; - MP_RETURN_IF_ERROR( - GlShader::CompileShader(GL_COMPUTE_SHADER, decode_src, &decode_shader)); - MP_RETURN_IF_ERROR(GpuProgram::CreateWithShader( - decode_shader, &gpu_data_->decode_program)); - // Outputs - size_t decoded_boxes_length = num_boxes_ * num_coords_; - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - decoded_boxes_length, &gpu_data_->decoded_boxes_buffer)); - // Inputs - size_t raw_boxes_length = num_boxes_ * num_coords_; - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - raw_boxes_length, &gpu_data_->raw_boxes_buffer)); - size_t raw_anchors_length = num_boxes_ * kNumCoordsPerBox; - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - raw_anchors_length, &gpu_data_->raw_anchors_buffer)); - // Parameters - glUseProgram(gpu_data_->decode_program.id()); - glUniform4f(0, options_.x_scale(), options_.y_scale(), options_.w_scale(), - options_.h_scale()); - - // A shader to score detection boxes. - const std::string score_src = absl::Substitute( - R"( #version 310 es - -layout(local_size_x = 1, local_size_y = $0, local_size_z = 1) in; - -#define FLT_MAX 1.0e+37 - -shared float local_scores[$0]; - -layout(std430, binding = 0) writeonly buffer Output { - float data[]; -} scored_boxes; - -layout(std430, binding = 1) readonly buffer Input0 { - float data[]; -} raw_scores; - -uint num_classes = uint($0); -int apply_sigmoid = int($1); -int apply_clipping_thresh = int($2); -float clipping_thresh = float($3); -int ignore_class_0 = int($4); - -float optional_sigmoid(float x) { - if (apply_sigmoid == int(0)) return x; - if (apply_clipping_thresh == int(1)) { - x = clamp(x, -clipping_thresh, clipping_thresh); - } - x = 1.0 / (1.0 + exp(-x)); - return x; -} - -void main() { - uint g_idx = gl_GlobalInvocationID.x; // box idx - uint s_idx = gl_LocalInvocationID.y; // score/class idx - - // load all scores into shared memory - float score = raw_scores.data[g_idx * num_classes + s_idx]; - local_scores[s_idx] = optional_sigmoid(score); - memoryBarrierShared(); - barrier(); - - // find max score in shared memory - if (s_idx == uint(0)) { - float max_score = -FLT_MAX; - float max_class = -1.0; - for (int i=ignore_class_0; i max_score) { - max_score = local_scores[i]; - max_class = float(i); - } - } - scored_boxes.data[g_idx * uint(2) + uint(0)] = max_score; - scored_boxes.data[g_idx * uint(2) + uint(1)] = max_class; - } -})", - num_classes_, options_.sigmoid_score() ? 1 : 0, - options_.has_score_clipping_thresh() ? 1 : 0, - options_.has_score_clipping_thresh() ? options_.score_clipping_thresh() - : 0, - !ignore_classes_.empty() ? 1 : 0); - - // # filter classes supported is hardware dependent. - int max_wg_size; // typically <= 1024 - glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1, - &max_wg_size); // y-dim - CHECK_LT(num_classes_, max_wg_size) - << "# classes must be < " << max_wg_size; - // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; - - // Shader program - GlShader score_shader; - MP_RETURN_IF_ERROR( - GlShader::CompileShader(GL_COMPUTE_SHADER, score_src, &score_shader)); - MP_RETURN_IF_ERROR( - GpuProgram::CreateWithShader(score_shader, &gpu_data_->score_program)); - // Outputs - size_t scored_boxes_length = num_boxes_ * 2; // score, class - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - scored_boxes_length, &gpu_data_->scored_boxes_buffer)); - // Inputs - size_t raw_scores_length = num_boxes_ * num_classes_; - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - raw_scores_length, &gpu_data_->raw_scores_buffer)); - - return absl::OkStatus(); - })); - -#elif MEDIAPIPE_TFLITE_METAL_INFERENCE - - gpu_data_ = absl::make_unique(); - id device = gpu_helper_.mtlDevice; - - // A shader to decode detection boxes. - std::string decode_src = absl::Substitute( - R"( -#include - -using namespace metal; - -kernel void decodeKernel( - device float* boxes [[ buffer(0) ]], - device float* raw_boxes [[ buffer(1) ]], - device float* raw_anchors [[ buffer(2) ]], - uint2 gid [[ thread_position_in_grid ]]) { - - uint num_coords = uint($0); - int reverse_output_order = int($1); - int apply_exponential = int($2); - int box_coord_offset = int($3); - int num_keypoints = int($4); - int keypt_coord_offset = int($5); - int num_values_per_keypt = int($6); -)", - options_.num_coords(), // box xywh - options_.reverse_output_order() ? 1 : 0, - options_.apply_exponential_on_box_size() ? 1 : 0, - options_.box_coord_offset(), options_.num_keypoints(), - options_.keypoint_coord_offset(), options_.num_values_per_keypoint()); - decode_src += absl::Substitute( - R"( - float4 scale = float4(($0),($1),($2),($3)); -)", - options_.x_scale(), options_.y_scale(), options_.w_scale(), - options_.h_scale()); - decode_src += R"( - uint g_idx = gid.x; - uint box_offset = g_idx * num_coords + uint(box_coord_offset); - uint anchor_offset = g_idx * uint(4); // check kNumCoordsPerBox - - float y_center, x_center, h, w; - - if (reverse_output_order == int(0)) { - y_center = raw_boxes[box_offset + uint(0)]; - x_center = raw_boxes[box_offset + uint(1)]; - h = raw_boxes[box_offset + uint(2)]; - w = raw_boxes[box_offset + uint(3)]; - } else { - x_center = raw_boxes[box_offset + uint(0)]; - y_center = raw_boxes[box_offset + uint(1)]; - w = raw_boxes[box_offset + uint(2)]; - h = raw_boxes[box_offset + uint(3)]; - } - - float anchor_yc = raw_anchors[anchor_offset + uint(0)]; - float anchor_xc = raw_anchors[anchor_offset + uint(1)]; - float anchor_h = raw_anchors[anchor_offset + uint(2)]; - float anchor_w = raw_anchors[anchor_offset + uint(3)]; - - x_center = x_center / scale.x * anchor_w + anchor_xc; - y_center = y_center / scale.y * anchor_h + anchor_yc; - - if (apply_exponential == int(1)) { - h = exp(h / scale.w) * anchor_h; - w = exp(w / scale.z) * anchor_w; - } else { - h = (h / scale.w) * anchor_h; - w = (w / scale.z) * anchor_w; - } - - float ymin = y_center - h / 2.0; - float xmin = x_center - w / 2.0; - float ymax = y_center + h / 2.0; - float xmax = x_center + w / 2.0; - - boxes[box_offset + uint(0)] = ymin; - boxes[box_offset + uint(1)] = xmin; - boxes[box_offset + uint(2)] = ymax; - boxes[box_offset + uint(3)] = xmax; - - if (num_keypoints > int(0)){ - for (int k = 0; k < num_keypoints; ++k) { - int kp_offset = - int(g_idx * num_coords) + keypt_coord_offset + k * num_values_per_keypt; - float kp_y, kp_x; - if (reverse_output_order == int(0)) { - kp_y = raw_boxes[kp_offset + int(0)]; - kp_x = raw_boxes[kp_offset + int(1)]; - } else { - kp_x = raw_boxes[kp_offset + int(0)]; - kp_y = raw_boxes[kp_offset + int(1)]; - } - boxes[kp_offset + int(0)] = kp_x / scale.x * anchor_w + anchor_xc; - boxes[kp_offset + int(1)] = kp_y / scale.y * anchor_h + anchor_yc; - } - } -})"; - - { - // Shader program - NSString* library_source = - [NSString stringWithUTF8String:decode_src.c_str()]; - NSError* error = nil; - id library = [device newLibraryWithSource:library_source - options:nullptr - error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"decodeKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - gpu_data_->decode_program = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(gpu_data_->decode_program != nil) - << "Couldn't create pipeline state " - << [[error localizedDescription] UTF8String]; - // Outputs - size_t decoded_boxes_length = num_boxes_ * num_coords_ * sizeof(float); - gpu_data_->decoded_boxes_buffer = - [device newBufferWithLength:decoded_boxes_length - options:MTLResourceStorageModeShared]; - // Inputs - size_t raw_boxes_length = num_boxes_ * num_coords_ * sizeof(float); - gpu_data_->raw_boxes_buffer = - [device newBufferWithLength:raw_boxes_length - options:MTLResourceStorageModeShared]; - size_t raw_anchors_length = num_boxes_ * kNumCoordsPerBox * sizeof(float); - gpu_data_->raw_anchors_buffer = - [device newBufferWithLength:raw_anchors_length - options:MTLResourceStorageModeShared]; - } - - // A shader to score detection boxes. - const std::string score_src = absl::Substitute( - R"( -#include - -using namespace metal; - -float optional_sigmoid(float x) { - int apply_sigmoid = int($1); - int apply_clipping_thresh = int($2); - float clipping_thresh = float($3); - if (apply_sigmoid == int(0)) return x; - if (apply_clipping_thresh == int(1)) { - x = clamp(x, -clipping_thresh, clipping_thresh); - } - x = 1.0 / (1.0 + exp(-x)); - return x; -} - -kernel void scoreKernel( - device float* scored_boxes [[ buffer(0) ]], - device float* raw_scores [[ buffer(1) ]], - uint2 tid [[ thread_position_in_threadgroup ]], - uint2 gid [[ thread_position_in_grid ]]) { - - uint num_classes = uint($0); - int apply_sigmoid = int($1); - int apply_clipping_thresh = int($2); - float clipping_thresh = float($3); - int ignore_class_0 = int($4); - - uint g_idx = gid.x; // box idx - uint s_idx = tid.y; // score/class idx - - // load all scores into shared memory - threadgroup float local_scores[$0]; - float score = raw_scores[g_idx * num_classes + s_idx]; - local_scores[s_idx] = optional_sigmoid(score); - threadgroup_barrier(mem_flags::mem_threadgroup); - - // find max score in shared memory - if (s_idx == uint(0)) { - float max_score = -FLT_MAX; - float max_class = -1.0; - for (int i=ignore_class_0; i max_score) { - max_score = local_scores[i]; - max_class = float(i); - } - } - scored_boxes[g_idx * uint(2) + uint(0)] = max_score; - scored_boxes[g_idx * uint(2) + uint(1)] = max_class; - } -})", - num_classes_, options_.sigmoid_score() ? 1 : 0, - options_.has_score_clipping_thresh() ? 1 : 0, - options_.has_score_clipping_thresh() ? options_.score_clipping_thresh() - : 0, - ignore_classes_.size() ? 1 : 0); - - // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; - - { - // Shader program - NSString* library_source = - [NSString stringWithUTF8String:score_src.c_str()]; - NSError* error = nil; - id library = [device newLibraryWithSource:library_source - options:nullptr - error:&error]; - RET_CHECK(library != nil) << "Couldn't create shader library " - << [[error localizedDescription] UTF8String]; - id kernel_func = nil; - kernel_func = [library newFunctionWithName:@"scoreKernel"]; - RET_CHECK(kernel_func != nil) << "Couldn't create kernel function."; - gpu_data_->score_program = - [device newComputePipelineStateWithFunction:kernel_func error:&error]; - RET_CHECK(gpu_data_->score_program != nil) - << "Couldn't create pipeline state " - << [[error localizedDescription] UTF8String]; - // Outputs - size_t scored_boxes_length = num_boxes_ * 2 * sizeof(float); // score,class - gpu_data_->scored_boxes_buffer = - [device newBufferWithLength:scored_boxes_length - options:MTLResourceStorageModeShared]; - // Inputs - size_t raw_scores_length = num_boxes_ * num_classes_ * sizeof(float); - gpu_data_->raw_scores_buffer = - [device newBufferWithLength:raw_scores_length - options:MTLResourceStorageModeShared]; - // # filter classes supported is hardware dependent. - int max_wg_size = gpu_data_->score_program.maxTotalThreadsPerThreadgroup; - CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size; - } - -#endif // MEDIAPIPE_TFLITE_GL_INFERENCE - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto deleted file mode 100644 index ef494c2cc..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TfLiteTensorsToDetectionsCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TfLiteTensorsToDetectionsCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TfLiteTensorsToDetectionsCalculatorOptions ext = 246514968; - } - - // The number of output classes predicted by the detection model. - required int32 num_classes = 1; - // The number of output boxes predicted by the detection model. - required int32 num_boxes = 2; - // The number of output values per boxes predicted by the detection model. The - // values contain bounding boxes, keypoints, etc. - required int32 num_coords = 3; - - // The offset of keypoint coordinates in the location tensor. - optional int32 keypoint_coord_offset = 9; - // The number of predicted keypoints. - optional int32 num_keypoints = 10 [default = 0]; - // The dimension of each keypoint, e.g. number of values predicted for each - // keypoint. - optional int32 num_values_per_keypoint = 11 [default = 2]; - // The offset of box coordinates in the location tensor. - optional int32 box_coord_offset = 12 [default = 0]; - - // Parameters for decoding SSD detection model. - optional float x_scale = 4 [default = 0.0]; - optional float y_scale = 5 [default = 0.0]; - optional float w_scale = 6 [default = 0.0]; - optional float h_scale = 7 [default = 0.0]; - - optional bool apply_exponential_on_box_size = 13 [default = false]; - - // Whether to reverse the order of predicted x, y from output. - // If false, the order is [y_center, x_center, h, w], if true the order is - // [x_center, y_center, w, h]. - optional bool reverse_output_order = 14 [default = false]; - // The ids of classes that should be ignored during decoding the score for - // each predicted box. - repeated int32 ignore_classes = 8; - - optional bool sigmoid_score = 15 [default = false]; - optional float score_clipping_thresh = 16; - - // Whether the detection coordinates from the input tensors should be flipped - // vertically (along the y-direction). This is useful, for example, when the - // input tensors represent detections defined with a coordinate system where - // the origin is at the top-left corner, whereas the desired detection - // representation has a bottom-left origin (e.g., in OpenGL). - optional bool flip_vertically = 18 [default = false]; - - // Score threshold for perserving decoded detections. - optional float min_score_thresh = 19; -} diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_floats_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_floats_calculator.cc deleted file mode 100644 index ef2946c32..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_floats_calculator.cc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { - -// A calculator for converting TFLite tensors to to a float or a float vector. -// -// Input: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first -// tensor will be used. -// Output: -// FLOAT(optional) - Converted single float number. -// FLOATS(optional) - Converted float vector. -// -// Notes: To output FLOAT stream, the input TFLite tensor must have size 1, e.g. -// only 1 float number in the tensor. -// -// Usage example: -// node { -// calculator: "TfLiteTensorsToFloatsCalculator" -// input_stream: "TENSORS:tensors" -// output_stream: "FLOATS:floats" -// } -class TfLiteTensorsToFloatsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(TfLiteTensorsToFloatsCalculator); - -absl::Status TfLiteTensorsToFloatsCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("TENSORS")); - RET_CHECK(cc->Outputs().HasTag("FLOATS") || cc->Outputs().HasTag("FLOAT")); - - cc->Inputs().Tag("TENSORS").Set>(); - if (cc->Outputs().HasTag("FLOATS")) { - cc->Outputs().Tag("FLOATS").Set>(); - } - if (cc->Outputs().HasTag("FLOAT")) { - cc->Outputs().Tag("FLOAT").Set(); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToFloatsCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToFloatsCalculator::Process(CalculatorContext* cc) { - RET_CHECK(!cc->Inputs().Tag("TENSORS").IsEmpty()); - - const auto& input_tensors = - cc->Inputs().Tag("TENSORS").Get>(); - // TODO: Add option to specify which tensor to take from. - const TfLiteTensor* raw_tensor = &input_tensors[0]; - const float* raw_floats = raw_tensor->data.f; - int num_values = 1; - for (int i = 0; i < raw_tensor->dims->size; ++i) { - RET_CHECK_GT(raw_tensor->dims->data[i], 0); - num_values *= raw_tensor->dims->data[i]; - } - - if (cc->Outputs().HasTag("FLOAT")) { - // TODO: Could add an index in the option to specifiy returning one - // value of a float array. - RET_CHECK_EQ(num_values, 1); - cc->Outputs().Tag("FLOAT").AddPacket( - MakePacket(raw_floats[0]).At(cc->InputTimestamp())); - } - if (cc->Outputs().HasTag("FLOATS")) { - auto output_floats = absl::make_unique>( - raw_floats, raw_floats + num_values); - cc->Outputs().Tag("FLOATS").Add(output_floats.release(), - cc->InputTimestamp()); - } - - return absl::OkStatus(); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc deleted file mode 100644 index 1be83bbe1..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ /dev/null @@ -1,282 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "tensorflow/lite/interpreter.h" - -namespace mediapipe { - -namespace { - -inline float Sigmoid(float value) { return 1.0f / (1.0f + std::exp(-value)); } - -float ApplyActivation( - ::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions::Activation - activation, - float value) { - switch (activation) { - case ::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions::SIGMOID: - return Sigmoid(value); - break; - default: - return value; - } -} - -} // namespace - -// A calculator for converting TFLite tensors from regression models into -// landmarks. Note that if the landmarks in the tensor has more than 5 -// dimensions, only the first 5 dimensions will be converted to -// [x,y,z, visibility, presence]. The latter two fields may also stay unset if -// such attributes are not supported in the model. -// -// Input: -// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first -// tensor will be used. The size of the values must be -// (num_dimension x num_landmarks). -// -// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or -// not. Overrides corresponding side packet and/or field in the calculator -// options. -// -// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. -// Overrides corresponding side packet and/or field in the calculator options. -// -// Input side packet: -// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or -// not. Overrides the corresponding field in the calculator options. -// -// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. -// Overrides the corresponding field in the calculator options. -// -// Output: -// LANDMARKS(optional) - Result MediaPipe landmarks. -// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks. -// -// Notes: -// To output normalized landmarks, user must provide the original input image -// size to the model using calculator option input_image_width and -// input_image_height. -// Usage example: -// node { -// calculator: "TfLiteTensorsToLandmarksCalculator" -// input_stream: "TENSORS:landmark_tensors" -// output_stream: "LANDMARKS:landmarks" -// output_stream: "NORM_LANDMARKS:landmarks" -// options: { -// [mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] { -// num_landmarks: 21 -// -// input_image_width: 256 -// input_image_height: 256 -// } -// } -// } -class TfLiteTensorsToLandmarksCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - int num_landmarks_ = 0; - bool flip_vertically_ = false; - bool flip_horizontally_ = false; - - ::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_; -}; -REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); - -absl::Status TfLiteTensorsToLandmarksCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - if (cc->Inputs().HasTag("TENSORS")) { - cc->Inputs().Tag("TENSORS").Set>(); - } - - if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) { - cc->Inputs().Tag("FLIP_HORIZONTALLY").Set(); - } - - if (cc->Inputs().HasTag("FLIP_VERTICALLY")) { - cc->Inputs().Tag("FLIP_VERTICALLY").Set(); - } - - if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { - cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set(); - } - - if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { - cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set(); - } - - if (cc->Outputs().HasTag("LANDMARKS")) { - cc->Outputs().Tag("LANDMARKS").Set(); - } - - if (cc->Outputs().HasTag("NORM_LANDMARKS")) { - cc->Outputs().Tag("NORM_LANDMARKS").Set(); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - if (cc->Outputs().HasTag("NORM_LANDMARKS")) { - RET_CHECK(options_.has_input_image_height() && - options_.has_input_image_width()) - << "Must provide input width/height for getting normalized landmarks."; - } - if (cc->Outputs().HasTag("LANDMARKS") && - (options_.flip_vertically() || options_.flip_horizontally() || - cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") || - cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) { - RET_CHECK(options_.has_input_image_height() && - options_.has_input_image_width()) - << "Must provide input width/height for using flip_vertically option " - "when outputing landmarks in absolute coordinates."; - } - - flip_horizontally_ = - cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") - ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get() - : options_.flip_horizontally(); - - flip_vertically_ = - cc->InputSidePackets().HasTag("FLIP_VERTICALLY") - ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get() - : options_.flip_vertically(); - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToLandmarksCalculator::Process( - CalculatorContext* cc) { - // Override values if specified so. - if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") && - !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) { - flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get(); - } - if (cc->Inputs().HasTag("FLIP_VERTICALLY") && - !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) { - flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get(); - } - - if (cc->Inputs().Tag("TENSORS").IsEmpty()) { - return absl::OkStatus(); - } - - const auto& input_tensors = - cc->Inputs().Tag("TENSORS").Get>(); - - const TfLiteTensor* raw_tensor = &input_tensors[0]; - - int num_values = 1; - for (int i = 0; i < raw_tensor->dims->size; ++i) { - num_values *= raw_tensor->dims->data[i]; - } - const int num_dimensions = num_values / num_landmarks_; - CHECK_GT(num_dimensions, 0); - - const float* raw_landmarks = raw_tensor->data.f; - - LandmarkList output_landmarks; - - for (int ld = 0; ld < num_landmarks_; ++ld) { - const int offset = ld * num_dimensions; - Landmark* landmark = output_landmarks.add_landmark(); - - if (flip_horizontally_) { - landmark->set_x(options_.input_image_width() - raw_landmarks[offset]); - } else { - landmark->set_x(raw_landmarks[offset]); - } - if (num_dimensions > 1) { - if (flip_vertically_) { - landmark->set_y(options_.input_image_height() - - raw_landmarks[offset + 1]); - } else { - landmark->set_y(raw_landmarks[offset + 1]); - } - } - if (num_dimensions > 2) { - landmark->set_z(raw_landmarks[offset + 2]); - } - if (num_dimensions > 3) { - landmark->set_visibility(ApplyActivation(options_.visibility_activation(), - raw_landmarks[offset + 3])); - } - if (num_dimensions > 4) { - landmark->set_presence(ApplyActivation(options_.presence_activation(), - raw_landmarks[offset + 4])); - } - } - - // Output normalized landmarks if required. - if (cc->Outputs().HasTag("NORM_LANDMARKS")) { - NormalizedLandmarkList output_norm_landmarks; - for (int i = 0; i < output_landmarks.landmark_size(); ++i) { - const Landmark& landmark = output_landmarks.landmark(i); - NormalizedLandmark* norm_landmark = output_norm_landmarks.add_landmark(); - norm_landmark->set_x(landmark.x() / options_.input_image_width()); - norm_landmark->set_y(landmark.y() / options_.input_image_height()); - // Scale Z coordinate as X + allow additional uniform normalization. - norm_landmark->set_z(landmark.z() / options_.input_image_width() / - options_.normalize_z()); - if (landmark.has_visibility()) { // Set only if supported in the model. - norm_landmark->set_visibility(landmark.visibility()); - } - if (landmark.has_presence()) { // Set only if supported in the model. - norm_landmark->set_presence(landmark.presence()); - } - } - cc->Outputs() - .Tag("NORM_LANDMARKS") - .AddPacket(MakePacket(output_norm_landmarks) - .At(cc->InputTimestamp())); - } - - // Output absolute landmarks. - if (cc->Outputs().HasTag("LANDMARKS")) { - cc->Outputs() - .Tag("LANDMARKS") - .AddPacket(MakePacket(output_landmarks) - .At(cc->InputTimestamp())); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToLandmarksCalculator::LoadOptions( - CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = - cc->Options<::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions>(); - num_landmarks_ = options_.num_landmarks(); - - return absl::OkStatus(); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.proto deleted file mode 100644 index 793639a53..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.proto +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The option proto for the TfLiteTensorsToLandmarksCalculator. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TfLiteTensorsToLandmarksCalculatorOptions { - extend .mediapipe.CalculatorOptions { - optional TfLiteTensorsToLandmarksCalculatorOptions ext = 257405002; - } - - enum Activation { - NONE = 0; - SIGMOID = 1; - } - - // Number of landmarks from the output of the model. - required int32 num_landmarks = 1; - - // Size of the input image for the model. These options are used only when - // normalized landmarks are needed. Z coordinate is scaled as X assuming - // a weak perspective projection camera model. - optional int32 input_image_width = 2; - optional int32 input_image_height = 3; - - // Whether the detection coordinates from the input tensors should be flipped - // vertically (along the y-direction). This is useful, for example, when the - // input tensors represent detections defined with a coordinate system where - // the origin is at the top-left corner, whereas the desired detection - // representation has a bottom-left origin (e.g., in OpenGL). - optional bool flip_vertically = 4 [default = false]; - - // Whether the detection coordinates from the input tensors should be flipped - // horizontally (along the x-direction). This is useful, for example, when the - // input image is horizontally flipped in ImageTransformationCalculator - // beforehand. - optional bool flip_horizontally = 6 [default = false]; - - // A value that Z coordinates should be divided by. This option is used only - // when normalized landmarks are needed. It is applied in addition to Z - // coordinate being re-scaled as X. - optional float normalize_z = 5 [default = 1.0]; - - // Apply activation function to the tensor representing landmark visibility. - optional Activation visibility_activation = 7 [default = NONE]; - - // Apply activation function to the tensor representing landmark presence. - optional Activation presence_activation = 8 [default = NONE]; -} diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc deleted file mode 100644 index 22a9a8d70..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.cc +++ /dev/null @@ -1,705 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/strings/str_format.h" -#include "absl/types/span.h" -#include "mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/shader_util.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_program.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl/gl_texture.h" -#include "tensorflow/lite/delegates/gpu/gl_delegate.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace { -constexpr int kWorkgroupSize = 8; // Block size for GPU shader. -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} -float Clamp(float val, float min, float max) { - return std::min(std::max(val, min), max); -} - -constexpr char kTensorsTag[] = "TENSORS"; -constexpr char kTensorsGpuTag[] = "TENSORS_GPU"; -constexpr char kSizeImageTag[] = "REFERENCE_IMAGE"; -constexpr char kSizeImageGpuTag[] = "REFERENCE_IMAGE_GPU"; -constexpr char kMaskTag[] = "MASK"; -constexpr char kMaskGpuTag[] = "MASK_GPU"; -constexpr char kPrevMaskTag[] = "PREV_MASK"; -constexpr char kPrevMaskGpuTag[] = "PREV_MASK_GPU"; - -} // namespace - -namespace mediapipe { - -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) -using ::tflite::gpu::gl::CopyBuffer; -using ::tflite::gpu::gl::CreateReadWriteRgbaImageTexture; -using ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer; -using ::tflite::gpu::gl::GlBuffer; -using ::tflite::gpu::gl::GlProgram; -using ::tflite::gpu::gl::GlShader; -#endif // !MEDIAPIPE_DISABLE_GPU - -// Converts TFLite tensors from a tflite segmentation model to an image mask. -// -// Performs optional upscale to REFERENCE_IMAGE dimensions if provided, -// otherwise the mask is the same size as input tensor. -// -// Produces result as an RGBA image, with the mask in both R & A channels. The -// value of each pixel is the probability of the specified class after softmax, -// scaled to 255 on CPU. The class can be specified through the -// |output_layer_index| option. -// -// Inputs: -// One of the following TENSORS tags: -// TENSORS: Vector of TfLiteTensor of type kTfLiteFloat32. -// The tensor dimensions are specified in this calculator's options. -// TENSORS_GPU: Vector of GlBuffer. -// One of the following REFERENCE_IMAGE tags: -// REFERENCE_IMAGE (optional): An ImageFrame input image, -// used only for output dimensions. -// REFERENCE_IMAGE_GPU (optional): A GpuBuffer input image, -// used only for output dimensions. -// One of the following PREV_MASK tags: -// PREV_MASK (optional): An ImageFrame input mask, Gray, RGB or RGBA, [0-255]. -// PREV_MASK_GPU (optional): A GpuBuffer input mask, RGBA, [0-1]. -// Output: -// One of the following MASK tags: -// MASK: An ImageFrame output mask, RGBA. -// MASK_GPU: A GpuBuffer output mask, RGBA. -// -// Options: -// See tflite_segmentation_calculator.proto -// -// Usage example: -// node { -// calculator: "TfLiteTensorsToSegmentationCalculator" -// input_stream: "TENSORS_GPU:tensors" -// input_stream: "IMAGE_GPU:input_video" -// output_stream: "MASK_GPU:hair_mask" -// node_options: { -// [mediapipe.TfLiteTensorsToSegmentationCalculatorOptions] { -// tensor_in_width: 512 -// tensor_in_height: 512 -// tensor_in_channels: 2 -// combine_with_previous_ratio: 1.0 -// output_layer_index: 1 -// } -// } -// } -// -class TfLiteTensorsToSegmentationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status LoadOptions(CalculatorContext* cc); - absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc); - absl::Status ProcessCpu(CalculatorContext* cc); - void GlRender(); - - ::mediapipe::TfLiteTensorsToSegmentationCalculatorOptions options_; - - int tensor_width_ = 0; - int tensor_height_ = 0; - int tensor_channels_ = 0; - - bool use_gpu_ = false; -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - mediapipe::GlCalculatorHelper gpu_helper_; - std::unique_ptr mask_program_with_prev_; - std::unique_ptr mask_program_no_prev_; - std::unique_ptr tensor_buffer_; - GLuint upsample_program_; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(TfLiteTensorsToSegmentationCalculator); - -// static -absl::Status TfLiteTensorsToSegmentationCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(!cc->Inputs().GetTags().empty()); - RET_CHECK(!cc->Outputs().GetTags().empty()); - - bool use_gpu = false; - - // Inputs CPU. - if (cc->Inputs().HasTag(kTensorsTag)) { - cc->Inputs().Tag(kTensorsTag).Set>(); - } - if (cc->Inputs().HasTag(kPrevMaskTag)) { - cc->Inputs().Tag(kPrevMaskTag).Set(); - } - if (cc->Inputs().HasTag(kSizeImageTag)) { - cc->Inputs().Tag(kSizeImageTag).Set(); - } - - // Inputs GPU. -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - if (cc->Inputs().HasTag(kTensorsGpuTag)) { - cc->Inputs().Tag(kTensorsGpuTag).Set>(); - use_gpu |= true; - } - if (cc->Inputs().HasTag(kPrevMaskGpuTag)) { - cc->Inputs().Tag(kPrevMaskGpuTag).Set(); - use_gpu |= true; - } - if (cc->Inputs().HasTag(kSizeImageGpuTag)) { - cc->Inputs().Tag(kSizeImageGpuTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - - // Outputs. - if (cc->Outputs().HasTag(kMaskTag)) { - cc->Outputs().Tag(kMaskTag).Set(); - } -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - if (cc->Outputs().HasTag(kMaskGpuTag)) { - cc->Outputs().Tag(kMaskGpuTag).Set(); - use_gpu |= true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - - if (use_gpu) { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToSegmentationCalculator::Open( - CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kTensorsGpuTag)) { - use_gpu_ = true; -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - MP_RETURN_IF_ERROR(LoadOptions(cc)); - - if (use_gpu_) { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(InitGpu(cc)); - return absl::OkStatus(); - })); -#else - RET_CHECK_FAIL() << "GPU processing not enabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToSegmentationCalculator::Process( - CalculatorContext* cc) { - if (use_gpu_) { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(ProcessGpu(cc)); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - MP_RETURN_IF_ERROR(ProcessCpu(cc)); - } - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToSegmentationCalculator::Close( - CalculatorContext* cc) { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - gpu_helper_.RunInGlContext([this] { - if (upsample_program_) glDeleteProgram(upsample_program_); - upsample_program_ = 0; - mask_program_with_prev_.reset(); - mask_program_no_prev_.reset(); - tensor_buffer_.reset(); - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToSegmentationCalculator::ProcessCpu( - CalculatorContext* cc) { - if (cc->Inputs().Tag(kTensorsTag).IsEmpty()) { - return absl::OkStatus(); - } - - // Get input streams. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskTag) && - !cc->Inputs().Tag(kPrevMaskTag).IsEmpty(); - const ImageFrame placeholder; - const auto& input_mask = - has_prev_mask ? cc->Inputs().Tag(kPrevMaskTag).Get() - : placeholder; - int output_width = tensor_width_, output_height = tensor_height_; - if (cc->Inputs().HasTag(kSizeImageTag)) { - const auto& input_image = cc->Inputs().Tag(kSizeImageTag).Get(); - output_width = input_image.Width(); - output_height = input_image.Height(); - } - RET_CHECK_EQ(input_tensors.size(), 1); - - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width_, tensor_height_), CV_8UC4); - - // Get input previous mask. - cv::Mat input_mask_mat; - if (has_prev_mask) { - cv::Mat temp_mask_mat = formats::MatView(&input_mask); - if (temp_mask_mat.channels() != 4) { - cv::Mat converted_mat; - cv::cvtColor(temp_mask_mat, converted_mat, - temp_mask_mat.channels() == 1 ? cv::COLOR_GRAY2RGBA - : cv::COLOR_RGB2RGBA); - temp_mask_mat = converted_mat.clone(); - } - cv::resize(temp_mask_mat, input_mask_mat, small_mask_mat.size()); - } - - // Copy input tensor. - const TfLiteTensor* raw_input_tensor = &input_tensors[0]; - const float* raw_input_data = raw_input_tensor->data.f; - cv::Mat tensor_mat(cv::Size(tensor_width_, tensor_height_), - CV_MAKETYPE(CV_32F, tensor_channels_)); - float* tensor_mat_ptr = tensor_mat.ptr(); - memcpy(tensor_mat_ptr, raw_input_data, raw_input_tensor->bytes); - - // Process mask tensor. - // Run softmax over tensor output and blend with previous mask. - const int output_layer_index = options_.output_layer_index(); - const float combine_with_prev_ratio = options_.combine_with_previous_ratio(); - for (int i = 0; i < tensor_height_; ++i) { - for (int j = 0; j < tensor_width_; ++j) { - // Only two channel input tensor is supported. - const cv::Vec2f input_pix = tensor_mat.at(i, j); - const float shift = std::max(input_pix[0], input_pix[1]); - const float softmax_denom = - std::exp(input_pix[0] - shift) + std::exp(input_pix[1] - shift); - float new_mask_value = - std::exp(input_pix[output_layer_index] - shift) / softmax_denom; - // Combine previous value with current using uncertainty^2 as mixing coeff - if (has_prev_mask) { - const float prev_mask_value = - input_mask_mat.at(i, j)[0] / 255.0f; - const float eps = 0.001; - float uncertainty_alpha = - 1.0 + - (new_mask_value * std::log(new_mask_value + eps) + - (1.0 - new_mask_value) * std::log(1.0 - new_mask_value + eps)) / - std::log(2.0f); - uncertainty_alpha = Clamp(uncertainty_alpha, 0.0f, 1.0f); - // Equivalent to: a = 1 - (1 - a) * (1 - a); (squaring the uncertainty) - uncertainty_alpha *= 2.0 - uncertainty_alpha; - const float mixed_mask_value = - new_mask_value * uncertainty_alpha + - prev_mask_value * (1.0f - uncertainty_alpha); - new_mask_value = mixed_mask_value * combine_with_prev_ratio + - (1.0f - combine_with_prev_ratio) * new_mask_value; - } - const uchar mask_value = static_cast(new_mask_value * 255); - // Set both R and A channels for convenience. - const cv::Vec4b out_value = {mask_value, 0, 0, mask_value}; - small_mask_mat.at(i, j) = out_value; - } - } - - if (options_.flip_vertically()) cv::flip(small_mask_mat, small_mask_mat, 0); - - // Upsample small mask into output. - cv::Mat large_mask_mat; - cv::resize(small_mask_mat, large_mask_mat, - cv::Size(output_width, output_height)); - - // Send out image as CPU packet. - std::unique_ptr output_mask = absl::make_unique( - ImageFormat::SRGBA, output_width, output_height); - cv::Mat output_mat = formats::MatView(output_mask.get()); - large_mask_mat.copyTo(output_mat); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -// Steps: -// 1. receive tensor and optional previous mask -// 2. process segmentation tensor into small mask -// 3. upsample small mask into output mask to be same size as input image -absl::Status TfLiteTensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc) { - if (cc->Inputs().Tag(kTensorsGpuTag).IsEmpty()) { - return absl::OkStatus(); - } -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - // Get input streams. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsGpuTag).Get>(); - const bool has_prev_mask = cc->Inputs().HasTag(kPrevMaskGpuTag) && - !cc->Inputs().Tag(kPrevMaskGpuTag).IsEmpty(); - const auto& input_mask = - has_prev_mask - ? cc->Inputs().Tag(kPrevMaskGpuTag).Get() - : mediapipe::GpuBuffer(); - int output_width = tensor_width_, output_height = tensor_height_; - if (cc->Inputs().HasTag(kSizeImageGpuTag)) { - const auto& input_image = - cc->Inputs().Tag(kSizeImageGpuTag).Get(); - output_width = input_image.width(); - output_height = input_image.height(); - } - RET_CHECK_EQ(input_tensors.size(), 1); - - // Create initial working mask texture. - ::tflite::gpu::gl::GlTexture small_mask_texture; - MP_RETURN_IF_ERROR(CreateReadWriteRgbaImageTexture( - tflite::gpu::DataType::UINT8, // GL_RGBA8 - {tensor_width_, tensor_height_}, &small_mask_texture)); - - // Get input previous mask. - auto input_mask_texture = has_prev_mask - ? gpu_helper_.CreateSourceTexture(input_mask) - : mediapipe::GlTexture(); - - // Copy input tensor. - MP_RETURN_IF_ERROR(CopyBuffer(input_tensors[0], *tensor_buffer_)); - - // Run shader, process mask tensor. - // Run softmax over tensor output and blend with previous mask. - { - const int output_index = 0; - glBindImageTexture(output_index, small_mask_texture.id(), 0, GL_FALSE, 0, - GL_WRITE_ONLY, GL_RGBA8); - MP_RETURN_IF_ERROR(tensor_buffer_->BindToIndex(2)); - - const tflite::gpu::uint3 workgroups = { - NumGroups(tensor_width_, kWorkgroupSize), - NumGroups(tensor_height_, kWorkgroupSize), 1}; - - if (!has_prev_mask) { - MP_RETURN_IF_ERROR(mask_program_no_prev_->Dispatch(workgroups)); - } else { - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_mask_texture.name()); - MP_RETURN_IF_ERROR(mask_program_with_prev_->Dispatch(workgroups)); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - } - } - - // Upsample small mask into output. - mediapipe::GlTexture output_texture = gpu_helper_.CreateDestinationTexture( - output_width, output_height, - mediapipe::GpuBufferFormat::kBGRA32); // actually GL_RGBA8 - - // Run shader, upsample result. - { - gpu_helper_.BindFramebuffer(output_texture); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, small_mask_texture.id()); - GlRender(); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } - - // Send out image as GPU packet. - auto output_image = output_texture.GetFrame(); - cc->Outputs() - .Tag(kMaskGpuTag) - .Add(output_image.release(), cc->InputTimestamp()); - - // Cleanup - input_mask_texture.Release(); - output_texture.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -void TfLiteTensorsToSegmentationCalculator::GlRender() { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(upsample_program_); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); -#endif // !MEDIAPIPE_DISABLE_GPU -} - -absl::Status TfLiteTensorsToSegmentationCalculator::LoadOptions( - CalculatorContext* cc) { - // Get calculator options specified in the graph. - options_ = - cc->Options<::mediapipe::TfLiteTensorsToSegmentationCalculatorOptions>(); - - if (!options_.has_tensor_width() || !options_.has_tensor_height() || - !options_.has_tensor_channels()) - RET_CHECK_FAIL() << "Missing tensor dimensions in options."; - - tensor_width_ = options_.tensor_width(); - tensor_height_ = options_.tensor_height(); - tensor_channels_ = options_.tensor_channels(); - RET_CHECK_EQ(tensor_channels_, 2) - << "Only 2 channel segmentation tensor currently supported"; - - return absl::OkStatus(); -} - -absl::Status TfLiteTensorsToSegmentationCalculator::InitGpu( - CalculatorContext* cc) { -#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> absl::Status { - // A shader to process a segmentation tensor into an output mask, - // and use an optional previous mask as input. - // Currently uses 4 channels for output, - // and sets both R and A channels as mask value. - const std::string shader_src_template = - R"( #version 310 es - -layout(local_size_x = $0, local_size_y = $0, local_size_z = 1) in; - -precision highp float; - -layout(std430, binding = 2) readonly buffer B0 { - vec2 elements[]; -} input_data; // data tensor -layout(binding = 1) uniform sampler2D input_texture; // previous mask -layout(rgba8, binding = 0) writeonly uniform highp image2D output_texture; - -uniform ivec2 out_size; - -const int output_layer_index = int($1); -const float combine_with_previous_ratio = float($2); - -// Will be replaced with either '#define READ_PREVIOUS' or empty std::string -$3 //DEFINE_READ_PREVIOUS - -void main() { - int out_width = out_size.x; - int out_height = out_size.y; - - ivec2 gid = ivec2(gl_GlobalInvocationID.xy); - if (gid.x >= out_width || gid.y >= out_height) { return; } - - int linear_index = gid.y * out_width + gid.x; - vec2 input_value = input_data.elements[linear_index]; - - // Only two channel input tensor is supported. - vec2 input_px = input_value.rg; - float shift = max(input_px.r, input_px.g); - float softmax_denom = exp(input_px.r - shift) + exp(input_px.g - shift); - float new_mask_value = - exp(input_px[output_layer_index] - shift) / softmax_denom; - - // Combine previous value with current using uncertainty^2 as mixing parameter -#ifdef READ_PREVIOUS - vec2 normalized_gid = vec2(gid) / vec2(out_width - 1, out_height - 1); - float prev_mask_value = texture(input_texture, normalized_gid).r; - - float eps = 0.001; - float uncertainty_alpha = - 1.0 + (new_mask_value * log(new_mask_value + eps) + - (1.0 - new_mask_value) * log(1.0 - new_mask_value + eps)) / - log(2.0f); - uncertainty_alpha = clamp(uncertainty_alpha, 0.0, 1.0); - // equivalent to a = 1 - (1 - a) * (1 - a); (squaring the uncertainty) - uncertainty_alpha *= 2.0 - uncertainty_alpha; - - float mixed_mask_value = new_mask_value * uncertainty_alpha + - prev_mask_value * (1.0f - uncertainty_alpha); - - // Use user provided value to mix raw value & a value mixed with previous mask - new_mask_value = mixed_mask_value * combine_with_previous_ratio + - (1.0f - combine_with_previous_ratio) * new_mask_value; -#endif // READ_PREVIOUS - - int y_coord = int($4); - ivec2 output_coordinate = ivec2(gid.x, y_coord); - // Set both R and A channels for convenience. - vec4 out_value = vec4(new_mask_value, 0.0, 0.0, new_mask_value); - imageStore(output_texture, output_coordinate, out_value); -})"; - - const std::string shader_src_no_previous = absl::Substitute( - shader_src_template, kWorkgroupSize, options_.output_layer_index(), - options_.combine_with_previous_ratio(), "", - options_.flip_vertically() ? "out_height - gid.y - 1" : "gid.y"); - const std::string shader_src_with_previous = absl::Substitute( - shader_src_template, kWorkgroupSize, options_.output_layer_index(), - options_.combine_with_previous_ratio(), "#define READ_PREVIOUS", - options_.flip_vertically() ? "out_height - gid.y - 1" : "gid.y"); - - // Shader programs. - GlShader shader_without_previous; - MP_RETURN_IF_ERROR(GlShader::CompileShader( - GL_COMPUTE_SHADER, shader_src_no_previous, &shader_without_previous)); - mask_program_no_prev_ = absl::make_unique(); - MP_RETURN_IF_ERROR(GlProgram::CreateWithShader( - shader_without_previous, mask_program_no_prev_.get())); - GlShader shader_with_previous; - MP_RETURN_IF_ERROR(GlShader::CompileShader( - GL_COMPUTE_SHADER, shader_src_with_previous, &shader_with_previous)); - mask_program_with_prev_ = absl::make_unique(); - MP_RETURN_IF_ERROR(GlProgram::CreateWithShader( - shader_with_previous, mask_program_with_prev_.get())); - - // Buffer storage for input tensor. - size_t tensor_length = tensor_width_ * tensor_height_ * tensor_channels_; - tensor_buffer_ = absl::make_unique(); - MP_RETURN_IF_ERROR(CreateReadWriteShaderStorageBuffer( - tensor_length, tensor_buffer_.get())); - - // Parameters. - glUseProgram(mask_program_with_prev_->id()); - glUniform2i(glGetUniformLocation(mask_program_with_prev_->id(), "out_size"), - tensor_width_, tensor_height_); - glUniform1i( - glGetUniformLocation(mask_program_with_prev_->id(), "input_texture"), - 1); - glUseProgram(mask_program_no_prev_->id()); - glUniform2i(glGetUniformLocation(mask_program_no_prev_->id(), "out_size"), - tensor_width_, tensor_height_); - glUniform1i( - glGetUniformLocation(mask_program_no_prev_->id(), "input_texture"), 1); - - // Vertex shader attributes. - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Simple pass-through shader, used for hardware upsampling. - std::string upsample_shader_base = R"( - #if __VERSION__ < 130 - #define in varying - #endif // __VERSION__ < 130 - - #ifdef GL_ES - #define fragColor gl_FragColor - precision highp float; - #else - #define lowp - #define mediump - #define highp - #define texture2D texture - out vec4 fragColor; - #endif // defined(GL_ES) - - in vec2 sample_coordinate; - uniform sampler2D input_data; - - void main() { - vec4 pix = texture2D(input_data, sample_coordinate); - fragColor = pix; - } -)"; - - // Program - mediapipe::GlhCreateProgram( - mediapipe::kBasicVertexShader, upsample_shader_base.c_str(), - NUM_ATTRIBUTES, &attr_name[0], attr_location, &upsample_program_); - RET_CHECK(upsample_program_) << "Problem initializing the program."; - - // Parameters - glUseProgram(upsample_program_); - glUniform1i(glGetUniformLocation(upsample_program_, "input_data"), 1); - - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.proto deleted file mode 100644 index d04aa562b..000000000 --- a/mediapipe/calculators/tflite/tflite_tensors_to_segmentation_calculator.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TfLiteTensorsToSegmentationCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TfLiteTensorsToSegmentationCalculatorOptions ext = 252526026; - } - - // Dimensions of input segmentation tensor to process. - required int32 tensor_width = 1; - required int32 tensor_height = 2; - required int32 tensor_channels = 3; - - // How much to use previous mask when computing current one; range [0-1]. - // This is a tradeoff between responsiveness (0.0) and accuracy (1.0). - optional float combine_with_previous_ratio = 4 [default = 1.0]; - - // Model specific: Channel to use for processing tensor. - optional int32 output_layer_index = 5 [default = 1]; - - // Flip result image mask along y-axis. - optional bool flip_vertically = 6; -} diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD deleted file mode 100644 index 62455f01f..000000000 --- a/mediapipe/calculators/util/BUILD +++ /dev/null @@ -1,1316 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "alignment_points_to_rects_calculator", - srcs = ["alignment_points_to_rects_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/calculators/util:detections_to_rects_calculator", - "//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "annotation_overlay_calculator_proto", - srcs = ["annotation_overlay_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - ], -) - -mediapipe_proto_library( - name = "detection_label_id_to_text_calculator_proto", - srcs = ["detection_label_id_to_text_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "timed_box_list_id_to_label_calculator_proto", - srcs = ["timed_box_list_id_to_label_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "latency_proto", - srcs = ["latency.proto"], -) - -mediapipe_proto_library( - name = "non_max_suppression_calculator_proto", - srcs = ["non_max_suppression_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "packet_frequency_proto", - srcs = ["packet_frequency.proto"], -) - -mediapipe_proto_library( - name = "packet_frequency_calculator_proto", - srcs = ["packet_frequency_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "packet_latency_calculator_proto", - srcs = ["packet_latency_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "collection_has_min_size_calculator_proto", - srcs = ["collection_has_min_size_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "association_calculator_proto", - srcs = ["association_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "packet_frequency_calculator", - srcs = ["packet_frequency_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/calculators/util:packet_frequency_calculator_cc_proto", - "//mediapipe/calculators/util:packet_frequency_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/time", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_frequency_calculator_test", - size = "small", - srcs = ["packet_frequency_calculator_test.cc"], - deps = [ - ":packet_frequency_calculator", - "//mediapipe/calculators/util:packet_frequency_cc_proto", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - ], -) - -cc_library( - name = "packet_latency_calculator", - srcs = ["packet_latency_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/calculators/util:latency_cc_proto", - "//mediapipe/calculators/util:packet_latency_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/time", - ], - alwayslink = 1, -) - -cc_test( - name = "packet_latency_calculator_test", - size = "small", - srcs = ["packet_latency_calculator_test.cc"], - deps = [ - ":packet_latency_calculator", - "//mediapipe/calculators/util:latency_cc_proto", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/tool:simulation_clock", - "//mediapipe/framework/tool:simulation_clock_executor", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/time", - ], -) - -cc_library( - name = "clock_timestamp_calculator", - srcs = ["clock_timestamp_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/time", - ], - alwayslink = 1, -) - -cc_library( - name = "clock_latency_calculator", - srcs = ["clock_latency_calculator.cc"], - visibility = [ - "//visibility:public", - ], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/time", - ], - alwayslink = 1, -) - -cc_library( - name = "annotation_overlay_calculator", - srcs = ["annotation_overlay_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":annotation_overlay_calculator_cc_proto", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/util:color_cc_proto", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - "//mediapipe/util:annotation_renderer", - "//mediapipe/util:render_data_cc_proto", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:shader_util", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "detection_label_id_to_text_calculator", - srcs = ["detection_label_id_to_text_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":detection_label_id_to_text_calculator_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "//mediapipe/framework/port:status", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/util:resource_util", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "timed_box_list_id_to_label_calculator", - srcs = ["timed_box_list_id_to_label_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":timed_box_list_id_to_label_calculator_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "//mediapipe/framework/port:status", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/util/tracking:box_tracker_cc_proto", - "//mediapipe/util:resource_util", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "non_max_suppression_calculator", - srcs = ["non_max_suppression_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":non_max_suppression_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:rectangle", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "thresholding_calculator", - srcs = ["thresholding_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":thresholding_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "detection_to_landmarks_calculator", - srcs = ["detection_to_landmarks_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "landmarks_to_detection_calculator", - srcs = ["landmarks_to_detection_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":landmarks_to_detection_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "detections_to_rects_calculator", - srcs = [ - "detections_to_rects_calculator.cc", - ], - hdrs = [ - "detections_to_rects_calculator.h", - ], - visibility = ["//visibility:public"], - deps = [ - ":detections_to_rects_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/types:optional", - ], - alwayslink = 1, -) - -cc_library( - name = "rect_transformation_calculator", - srcs = ["rect_transformation_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":rect_transformation_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "rect_projection_calculator", - srcs = ["rect_projection_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "detections_to_rects_calculator_test", - size = "small", - srcs = ["detections_to_rects_calculator_test.cc"], - deps = [ - ":detections_to_rects_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -mediapipe_proto_library( - name = "rect_to_render_data_calculator_proto", - srcs = ["rect_to_render_data_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "rect_to_render_scale_calculator_proto", - srcs = ["rect_to_render_scale_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "detections_to_render_data_calculator_proto", - srcs = ["detections_to_render_data_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "landmarks_to_render_data_calculator_proto", - srcs = ["landmarks_to_render_data_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "timed_box_list_to_render_data_calculator_proto", - srcs = ["timed_box_list_to_render_data_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "labels_to_render_data_calculator_proto", - srcs = ["labels_to_render_data_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "thresholding_calculator_proto", - srcs = ["thresholding_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -mediapipe_proto_library( - name = "detections_to_rects_calculator_proto", - srcs = ["detections_to_rects_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "landmark_projection_calculator_proto", - srcs = ["landmark_projection_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "landmark_visibility_calculator", - srcs = ["landmark_visibility_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -cc_library( - name = "set_landmark_visibility_calculator", - srcs = ["set_landmark_visibility_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "landmarks_to_floats_calculator_proto", - srcs = ["landmarks_to_floats_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "rect_transformation_calculator_proto", - srcs = ["rect_transformation_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "landmarks_to_detection_calculator_proto", - srcs = ["landmarks_to_detection_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - "//mediapipe/util:color_proto", - "//mediapipe/util:render_data_proto", - ], -) - -cc_library( - name = "detections_to_render_data_calculator", - srcs = ["detections_to_render_data_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":detections_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "landmarks_to_render_data_calculator", - srcs = ["landmarks_to_render_data_calculator.cc"], - hdrs = ["landmarks_to_render_data_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":landmarks_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "timed_box_list_to_render_data_calculator", - srcs = ["timed_box_list_to_render_data_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":timed_box_list_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - "//mediapipe/util/tracking:box_tracker_cc_proto", - "//mediapipe/util/tracking:tracking_cc_proto", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "labels_to_render_data_calculator", - srcs = ["labels_to_render_data_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":labels_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "rect_to_render_data_calculator", - srcs = ["rect_to_render_data_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":rect_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - ], - alwayslink = 1, -) - -cc_library( - name = "rect_to_render_scale_calculator", - srcs = ["rect_to_render_scale_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":rect_to_render_scale_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - ], - alwayslink = 1, -) - -cc_test( - name = "detections_to_render_data_calculator_test", - size = "small", - srcs = ["detections_to_render_data_calculator_test.cc"], - deps = [ - ":detections_to_render_data_calculator", - ":detections_to_render_data_calculator_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "detection_letterbox_removal_calculator", - srcs = ["detection_letterbox_removal_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "detection_projection_calculator", - srcs = ["detection_projection_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:point", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "detection_projection_calculator_test", - srcs = ["detection_projection_calculator_test.cc"], - deps = [ - ":detection_projection_calculator", - "//mediapipe/calculators/tensor:image_to_tensor_utils", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:point", - ], -) - -cc_library( - name = "landmark_letterbox_removal_calculator", - srcs = ["landmark_letterbox_removal_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "landmark_projection_calculator", - srcs = ["landmark_projection_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":landmark_projection_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "landmark_projection_calculator_test", - srcs = ["landmark_projection_calculator_test.cc"], - deps = [ - ":landmark_projection_calculator", - "//mediapipe/calculators/tensor:image_to_tensor_utils", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/memory", - "@com_google_googletest//:gtest_main", - ], -) - -mediapipe_proto_library( - name = "landmarks_smoothing_calculator_proto", - srcs = ["landmarks_smoothing_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "landmarks_smoothing_calculator", - srcs = ["landmarks_smoothing_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":landmarks_smoothing_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util/filtering:one_euro_filter", - "//mediapipe/util/filtering:relative_velocity_filter", - "@com_google_absl//absl/algorithm:container", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "visibility_smoothing_calculator_proto", - srcs = ["visibility_smoothing_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "visibility_smoothing_calculator", - srcs = ["visibility_smoothing_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":visibility_smoothing_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util/filtering:low_pass_filter", - "@com_google_absl//absl/algorithm:container", - ], - alwayslink = 1, -) - -cc_library( - name = "landmarks_to_floats_calculator", - srcs = ["landmarks_to_floats_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":landmarks_to_floats_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_test( - name = "detection_letterbox_removal_calculator_test", - srcs = ["detection_letterbox_removal_calculator_test.cc"], - deps = [ - ":detection_letterbox_removal_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - ], -) - -cc_test( - name = "landmark_letterbox_removal_calculator_test", - srcs = ["landmark_letterbox_removal_calculator_test.cc"], - deps = [ - ":landmark_letterbox_removal_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:validate_type", - ], -) - -mediapipe_proto_library( - name = "top_k_scores_calculator_proto", - srcs = ["top_k_scores_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "top_k_scores_calculator", - srcs = ["top_k_scores_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":top_k_scores_calculator_cc_proto", - "@com_google_absl//absl/container:node_hash_map", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework:calculator_framework", - "//mediapipe/util:resource_util", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_test( - name = "top_k_scores_calculator_test", - srcs = ["top_k_scores_calculator_test.cc"], - deps = [ - ":top_k_scores_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -mediapipe_proto_library( - name = "local_file_contents_calculator_proto", - srcs = ["local_file_contents_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "local_file_contents_calculator", - srcs = ["local_file_contents_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":local_file_contents_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:resource_util", - ], - alwayslink = 1, -) - -cc_library( - name = "local_file_pattern_contents_calculator", - srcs = ["local_file_pattern_contents_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "filter_collection_calculator", - srcs = ["filter_collection_calculator.cc"], - hdrs = ["filter_collection_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "collection_has_min_size_calculator", - srcs = ["collection_has_min_size_calculator.cc"], - hdrs = ["collection_has_min_size_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":collection_has_min_size_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "collection_has_min_size_calculator_test", - srcs = ["collection_has_min_size_calculator_test.cc"], - deps = [ - ":collection_has_min_size_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_library( - name = "association_calculator", - hdrs = ["association_calculator.h"], - visibility = ["//visibility:public"], - deps = [ - ":association_calculator_cc_proto", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework/port:rectangle", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], - alwayslink = 1, -) - -cc_library( - name = "association_norm_rect_calculator", - srcs = ["association_norm_rect_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":association_calculator", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:rectangle", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "association_detection_calculator", - srcs = ["association_detection_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":association_calculator", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:rectangle", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_test( - name = "association_calculator_test", - srcs = ["association_calculator_test.cc"], - deps = [ - ":association_detection_calculator", - ":association_norm_rect_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework:collection_item_id", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_library( - name = "detections_to_timed_box_list_calculator", - srcs = ["detections_to_timed_box_list_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util/tracking:box_tracker", - ], - alwayslink = 1, -) - -cc_library( - name = "detection_unique_id_calculator", - srcs = ["detection_unique_id_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -mediapipe_proto_library( - name = "logic_calculator_proto", - srcs = ["logic_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "logic_calculator", - srcs = ["logic_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":logic_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -cc_library( - name = "to_image_calculator", - srcs = ["to_image_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "from_image_calculator", - srcs = ["from_image_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - ] + select({ - "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "//mediapipe/gpu:gl_calculator_helper", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "detection_classifications_merger_calculator", - srcs = ["detection_classifications_merger_calculator.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_test( - name = "detection_classifications_merger_calculator_test", - srcs = ["detection_classifications_merger_calculator_test.cc"], - deps = [ - ":detection_classifications_merger_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/formats:classification_cc_proto", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -mediapipe_proto_library( - name = "refine_landmarks_from_heatmap_calculator_proto", - srcs = ["refine_landmarks_from_heatmap_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -cc_library( - name = "refine_landmarks_from_heatmap_calculator", - srcs = ["refine_landmarks_from_heatmap_calculator.cc"], - hdrs = ["refine_landmarks_from_heatmap_calculator.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], - deps = [ - ":refine_landmarks_from_heatmap_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/formats:landmark_cc_proto", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:statusor", - ], - alwayslink = 1, -) - -cc_test( - name = "refine_landmarks_from_heatmap_calculator_test", - srcs = ["refine_landmarks_from_heatmap_calculator_test.cc"], - deps = [ - ":refine_landmarks_from_heatmap_calculator", - "//mediapipe/framework/port:gtest_main", - ], -) diff --git a/mediapipe/calculators/util/alignment_points_to_rects_calculator.cc b/mediapipe/calculators/util/alignment_points_to_rects_calculator.cc deleted file mode 100644 index edfa4196a..000000000 --- a/mediapipe/calculators/util/alignment_points_to_rects_calculator.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include - -#include "mediapipe/calculators/util/detections_to_rects_calculator.h" -#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace {} // namespace - -// A calculator that converts Detection with two alignment points to Rect. -// -// Detection should contain two points: -// * Center point - center of the crop -// * Scale point - vector from center to scale point defines size and rotation -// of the Rect. Not that Y coordinate of this vector is flipped before -// computing the rotation (it is caused by the fact that Y axis is -// directed downwards). So define target rotation vector accordingly. -// -// Example config: -// node { -// calculator: "AlignmentPointsRectsCalculator" -// input_stream: "DETECTIONS:detections" -// input_stream: "IMAGE_SIZE:image_size" -// output_stream: "NORM_RECT:rect" -// options: { -// [mediapipe.DetectionsToRectsCalculatorOptions.ext] { -// rotation_vector_start_keypoint_index: 0 -// rotation_vector_end_keypoint_index: 1 -// rotation_vector_target_angle_degrees: 90 -// output_zero_rect_for_empty_detections: true -// } -// } -// } -class AlignmentPointsRectsCalculator : public DetectionsToRectsCalculator { - public: - absl::Status Open(CalculatorContext* cc) override { - RET_CHECK_OK(DetectionsToRectsCalculator::Open(cc)); - - // Make sure that start and end keypoints are provided. - // They are required for the rect size calculation and will also force base - // calculator to compute rotation. - options_ = cc->Options(); - RET_CHECK(options_.has_rotation_vector_start_keypoint_index()) - << "Start keypoint is required to calculate rect size and rotation"; - RET_CHECK(options_.has_rotation_vector_end_keypoint_index()) - << "End keypoint is required to calculate rect size and rotation"; - - return absl::OkStatus(); - } - - private: - absl::Status DetectionToNormalizedRect( - const ::mediapipe::Detection& detection, - const DetectionSpec& detection_spec, - ::mediapipe::NormalizedRect* rect) override; -}; -REGISTER_CALCULATOR(AlignmentPointsRectsCalculator); - -absl::Status AlignmentPointsRectsCalculator::DetectionToNormalizedRect( - const Detection& detection, const DetectionSpec& detection_spec, - NormalizedRect* rect) { - const auto& location_data = detection.location_data(); - const auto& image_size = detection_spec.image_size; - RET_CHECK(image_size) << "Image size is required to calculate the rect"; - - const float x_center = - location_data.relative_keypoints(start_keypoint_index_).x() * - image_size->first; - const float y_center = - location_data.relative_keypoints(start_keypoint_index_).y() * - image_size->second; - - const float x_scale = - location_data.relative_keypoints(end_keypoint_index_).x() * - image_size->first; - const float y_scale = - location_data.relative_keypoints(end_keypoint_index_).y() * - image_size->second; - - // Bounding box size as double distance from center to scale point. - const float box_size = - std::sqrt((x_scale - x_center) * (x_scale - x_center) + - (y_scale - y_center) * (y_scale - y_center)) * - 2.0; - - // Set resulting bounding box. - rect->set_x_center(x_center / image_size->first); - rect->set_y_center(y_center / image_size->second); - rect->set_width(box_size / image_size->first); - rect->set_height(box_size / image_size->second); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc deleted file mode 100644 index 2c0b25397..000000000 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ /dev/null @@ -1,665 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/util/annotation_overlay_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" -#include "mediapipe/util/annotation_renderer.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/gpu/shader_util.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { - -namespace { - -constexpr char kVectorTag[] = "VECTOR"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kImageFrameTag[] = "IMAGE"; - -enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; - -// Round up n to next multiple of m. -size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT - -// When using GPU, this color will become transparent when the calculator -// merges the annotation overlay with the image frame. As a result, drawing in -// this color is not supported and it should be set to something unlikely used. -constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value. - -// Future Image type. -inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; } -} // namespace - -// A calculator for rendering data on images. -// -// Inputs: -// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer), -// containing the input image. -// If output is CPU, and input isn't provided, the renderer creates a -// blank canvas with the width, height and color provided in the options. -// 2. RenderData proto on variable number of input streams. All the RenderData -// at a particular timestamp is drawn on the image in the order of their -// input streams. No tags required. -// 3. std::vector on variable number of input streams. RenderData -// objects at a particular timestamp are drawn on the image in order of the -// input vector items. These input streams are tagged with "VECTOR". -// -// Output: -// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer), -// Note: Output types should match their corresponding input stream type. -// -// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The -// output format is the same as input except for GRAY8 where the output is in -// SRGB to support annotations in color. -// -// For GPU input frames, only 4-channel images are supported. -// -// Note: When using GPU, drawing with color kAnnotationBackgroundColor (defined -// above) is not supported. -// -// Example config (CPU): -// node { -// calculator: "AnnotationOverlayCalculator" -// input_stream: "IMAGE:image_frames" -// input_stream: "render_data_1" -// input_stream: "render_data_2" -// input_stream: "render_data_3" -// input_stream: "VECTOR:0:render_data_vec_0" -// input_stream: "VECTOR:1:render_data_vec_1" -// output_stream: "IMAGE:decorated_frames" -// options { -// [mediapipe.AnnotationOverlayCalculatorOptions.ext] { -// } -// } -// } -// -// Example config (GPU): -// node { -// calculator: "AnnotationOverlayCalculator" -// input_stream: "IMAGE_GPU:image_frames" -// input_stream: "render_data_1" -// input_stream: "render_data_2" -// input_stream: "render_data_3" -// input_stream: "VECTOR:0:render_data_vec_0" -// input_stream: "VECTOR:1:render_data_vec_1" -// output_stream: "IMAGE_GPU:decorated_frames" -// options { -// [mediapipe.AnnotationOverlayCalculatorOptions.ext] { -// } -// } -// } -// -class AnnotationOverlayCalculator : public CalculatorBase { - public: - AnnotationOverlayCalculator() = default; - ~AnnotationOverlayCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status CreateRenderTargetCpu(CalculatorContext* cc, - std::unique_ptr& image_mat, - ImageFormat::Format* target_format); - template - absl::Status CreateRenderTargetGpu(CalculatorContext* cc, - std::unique_ptr& image_mat); - template - absl::Status RenderToGpu(CalculatorContext* cc, uchar* overlay_image); - absl::Status RenderToCpu(CalculatorContext* cc, - const ImageFormat::Format& target_format, - uchar* data_image); - - absl::Status GlRender(CalculatorContext* cc); - template - absl::Status GlSetup(CalculatorContext* cc); - - // Options for the calculator. - AnnotationOverlayCalculatorOptions options_; - - // Underlying helper renderer library. - std::unique_ptr renderer_; - - // Indicates if image frame is available as input. - bool image_frame_available_ = false; - - bool use_gpu_ = false; - bool gpu_initialized_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; - GLuint program_ = 0; - GLuint image_mat_tex_ = 0; // Overlay drawing image for GPU. - int width_ = 0; - int height_ = 0; - int width_canvas_ = 0; // Size of overlay drawing texture canvas. - int height_canvas_ = 0; -#endif // MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(AnnotationOverlayCalculator); - -absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); - - bool use_gpu = false; - - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("Cannot have multiple input images."); - } - if (cc->Inputs().HasTag(kGpuBufferTag) != - cc->Outputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("GPU output must have GPU input."); - } - - // Input image to render onto copy of. Should be same type as output. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - cc->Inputs().Tag(kGpuBufferTag).Set(); - CHECK(cc->Outputs().HasTag(kGpuBufferTag)); - use_gpu = true; - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); - } - - // Data streams to render. - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); - ++id) { - auto tag_and_index = cc->Inputs().TagAndIndexFromId(id); - std::string tag = tag_and_index.first; - if (tag == kVectorTag) { - cc->Inputs().Get(id).Set>(); - } else if (tag.empty()) { - // Empty tag defaults to accepting a single object of RenderData type. - cc->Inputs().Get(id).Set(); - } - } - - // Rendered image. Should be same type as input. -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kGpuBufferTag)) { - cc->Outputs().Tag(kGpuBufferTag).Set(); - } -#endif // !MEDIAPIPE_DISABLE_GPU - if (cc->Outputs().HasTag(kImageFrameTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); - } - - if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - if (cc->Inputs().HasTag(kGpuBufferTag) || HasImageTag(cc)) { -#if !MEDIAPIPE_DISABLE_GPU - use_gpu_ = true; -#endif // !MEDIAPIPE_DISABLE_GPU - } - - if (cc->Inputs().HasTag(kGpuBufferTag) || - cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) { - image_frame_available_ = true; - } else { - RET_CHECK(options_.has_canvas_width_px()); - RET_CHECK(options_.has_canvas_height_px()); - } - - // Initialize the helper renderer library. - renderer_ = absl::make_unique(); - renderer_->SetFlipTextVertically(options_.flip_text_vertically()); - if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor()); - - // Set the output header based on the input header (if present). - const char* tag = use_gpu_ ? kGpuBufferTag : kImageFrameTag; - if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { - const auto& input_header = - cc->Inputs().Tag(tag).Header().Get(); - auto* output_video_header = new VideoHeader(input_header); - cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header)); - } - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { - // Initialize render target, drawn with OpenCV. - std::unique_ptr image_mat; - ImageFormat::Format target_format; - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - return GlSetup(cc); - })); - gpu_initialized_ = true; - } - if (cc->Inputs().HasTag(kGpuBufferTag)) { - MP_RETURN_IF_ERROR( - (CreateRenderTargetGpu( - cc, image_mat))); - } -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - if (cc->Outputs().HasTag(kImageFrameTag)) { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - } - } - - // Reset the renderer with the image_mat. No copy here. - renderer_->AdoptImage(image_mat.get()); - - // Render streams onto render target. - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); - ++id) { - auto tag_and_index = cc->Inputs().TagAndIndexFromId(id); - std::string tag = tag_and_index.first; - if (!tag.empty() && tag != kVectorTag) { - continue; - } - if (cc->Inputs().Get(id).IsEmpty()) { - continue; - } - if (tag.empty()) { - // Empty tag defaults to accepting a single object of RenderData type. - const RenderData& render_data = cc->Inputs().Get(id).Get(); - renderer_->RenderDataOnImage(render_data); - } else { - RET_CHECK_EQ(kVectorTag, tag); - const std::vector& render_data_vec = - cc->Inputs().Get(id).Get>(); - for (const RenderData& render_data : render_data_vec) { - renderer_->RenderDataOnImage(render_data); - } - } - } - - if (use_gpu_) { -#if !MEDIAPIPE_DISABLE_GPU - // Overlay rendered image in OpenGL, onto a copy of input. - uchar* image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, cc, image_mat_ptr]() -> absl::Status { - return RenderToGpu( - cc, image_mat_ptr); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - // Copy the rendered image to output. - uchar* image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); - } - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::Close(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - program_ = 0; - if (image_mat_tex_) glDeleteTextures(1, &image_mat_tex_); - image_mat_tex_ = 0; - }); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::RenderToCpu( - CalculatorContext* cc, const ImageFormat::Format& target_format, - uchar* data_image) { - auto output_frame = absl::make_unique( - target_format, renderer_->GetImageWidth(), renderer_->GetImageHeight()); - -#if !MEDIAPIPE_DISABLE_GPU - output_frame->CopyPixelData(target_format, renderer_->GetImageWidth(), - renderer_->GetImageHeight(), data_image, - ImageFrame::kGlDefaultAlignmentBoundary); -#else - output_frame->CopyPixelData(target_format, renderer_->GetImageWidth(), - renderer_->GetImageHeight(), data_image, - ImageFrame::kDefaultAlignmentBoundary); -#endif // !MEDIAPIPE_DISABLE_GPU - - if (cc->Outputs().HasTag(kImageFrameTag)) { - cc->Outputs() - .Tag(kImageFrameTag) - .Add(output_frame.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -template -absl::Status AnnotationOverlayCalculator::RenderToGpu(CalculatorContext* cc, - uchar* overlay_image) { -#if !MEDIAPIPE_DISABLE_GPU - // Source and destination textures. - const auto& input_frame = cc->Inputs().Tag(Tag).Get(); - auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); - - auto output_texture = gpu_helper_.CreateDestinationTexture( - width_, height_, mediapipe::GpuBufferFormat::kBGRA32); - - // Upload render target to GPU. - { - glBindTexture(GL_TEXTURE_2D, image_mat_tex_); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width_canvas_, height_canvas_, - GL_RGB, GL_UNSIGNED_BYTE, overlay_image); - glBindTexture(GL_TEXTURE_2D, 0); - } - - // Blend overlay image in GPU shader. - { - gpu_helper_.BindFramebuffer(output_texture); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, input_texture.name()); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, image_mat_tex_); - - MP_RETURN_IF_ERROR(GlRender(cc)); - - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, 0); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, 0); - glFlush(); - } - - // Send out blended image as GPU packet. - auto output_frame = output_texture.GetFrame(); - cc->Outputs().Tag(Tag).Add(output_frame.release(), cc->InputTimestamp()); - - // Cleanup - input_texture.Release(); - output_texture.Release(); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpu( - CalculatorContext* cc, std::unique_ptr& image_mat, - ImageFormat::Format* target_format) { - if (image_frame_available_) { - const auto& input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - if (input_frame.Format() == ImageFormat::GRAY8) { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } else { - input_mat.copyTo(*image_mat); - } - } else { - image_mat = absl::make_unique( - options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3, - cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(), - options_.canvas_color().b())); - *target_format = ImageFormat::SRGB; - } - - return absl::OkStatus(); -} - -template -absl::Status AnnotationOverlayCalculator::CreateRenderTargetGpu( - CalculatorContext* cc, std::unique_ptr& image_mat) { -#if !MEDIAPIPE_DISABLE_GPU - if (image_frame_available_) { - const auto& input_frame = cc->Inputs().Tag(Tag).Get(); - const mediapipe::ImageFormat::Format format = - mediapipe::ImageFormatForGpuBufferFormat(input_frame.format()); - if (format != mediapipe::ImageFormat::SRGBA && - format != mediapipe::ImageFormat::SRGB) - RET_CHECK_FAIL() << "Unsupported GPU input format: " << format; - image_mat = - absl::make_unique(height_canvas_, width_canvas_, CV_8UC3); - memset(image_mat->data, kAnnotationBackgroundColor, - height_canvas_ * width_canvas_ * image_mat->elemSize()); - } else { - image_mat = absl::make_unique( - height_canvas_, width_canvas_, CV_8UC3, - cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(), - options_.canvas_color().b())); - } -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status AnnotationOverlayCalculator::GlRender(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - static const GLfloat square_vertices[] = { - -1.0f, -1.0f, // bottom left - 1.0f, -1.0f, // bottom right - -1.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - static const GLfloat texture_vertices[] = { - 0.0f, 0.0f, // bottom left - 1.0f, 0.0f, // bottom right - 0.0f, 1.0f, // top left - 1.0f, 1.0f, // top right - }; - - // program - glUseProgram(program_); - - // vertex storage - GLuint vbo[2]; - glGenBuffers(2, vbo); - GLuint vao; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - - // vbo 0 - glBindBuffer(GL_ARRAY_BUFFER, vbo[0]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), square_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_VERTEX); - glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, nullptr); - - // vbo 1 - glBindBuffer(GL_ARRAY_BUFFER, vbo[1]); - glBufferData(GL_ARRAY_BUFFER, 4 * 2 * sizeof(GLfloat), texture_vertices, - GL_STATIC_DRAW); - glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glVertexAttribPointer(ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0, nullptr); - - // draw - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // cleanup - glDisableVertexAttribArray(ATTRIB_VERTEX); - glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION); - glBindBuffer(GL_ARRAY_BUFFER, 0); - glBindVertexArray(0); - glDeleteVertexArrays(1, &vao); - glDeleteBuffers(2, vbo); -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -template -absl::Status AnnotationOverlayCalculator::GlSetup(CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_GPU - const GLint attr_location[NUM_ATTRIBUTES] = { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - }; - const GLchar* attr_name[NUM_ATTRIBUTES] = { - "position", - "texture_coordinate", - }; - - // Shader to overlay a texture onto another when overlay is non-zero. - constexpr char kFragSrcBody[] = R"( - DEFAULT_PRECISION(mediump, float) - #ifdef GL_ES - #define fragColor gl_FragColor - #else - out vec4 fragColor; - #endif // GL_ES - - in vec2 sample_coordinate; - uniform sampler2D input_frame; - // "overlay" texture has top-left origin (OpenCV mat with annotations has - // been uploaded to GPU without vertical flip) - uniform sampler2D overlay; - uniform vec3 transparent_color; - - void main() { - vec3 image_pix = texture2D(input_frame, sample_coordinate).rgb; - #ifdef INPUT_FRAME_HAS_TOP_LEFT_ORIGIN - // "input_frame" has top-left origin same as "overlay", hence overlaying - // as is. - vec3 overlay_pix = texture2D(overlay, sample_coordinate).rgb; - #else - // "input_frame" has bottom-left origin, hence flipping "overlay" texture - // coordinates. - vec3 overlay_pix = texture2D(overlay, vec2(sample_coordinate.x, 1.0 - sample_coordinate.y)).rgb; - #endif // INPUT_FRAME_HAS_TOP_LEFT_ORIGIN - - vec3 out_pix = image_pix; - float dist = distance(overlay_pix.rgb, transparent_color); - if (dist > 0.001) out_pix = overlay_pix; - fragColor.rgb = out_pix; - fragColor.a = 1.0; - } - )"; - - std::string defines; - if (options_.gpu_uses_top_left_origin()) { - defines = R"( - #define INPUT_FRAME_HAS_TOP_LEFT_ORIGIN; - )"; - } - - const std::string frag_src = absl::StrCat( - mediapipe::kMediaPipeFragmentShaderPreamble, defines, kFragSrcBody); - - // Create shader program and set parameters - mediapipe::GlhCreateProgram(mediapipe::kBasicVertexShader, frag_src.c_str(), - NUM_ATTRIBUTES, (const GLchar**)&attr_name[0], - attr_location, &program_); - RET_CHECK(program_) << "Problem initializing the program."; - glUseProgram(program_); - glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); - glUniform1i(glGetUniformLocation(program_, "overlay"), 2); - glUniform3f(glGetUniformLocation(program_, "transparent_color"), - kAnnotationBackgroundColor / 255.0, - kAnnotationBackgroundColor / 255.0, - kAnnotationBackgroundColor / 255.0); - - // Ensure GPU texture is divisible by 4. See b/138751944 for more info. - const float alignment = ImageFrame::kGlDefaultAlignmentBoundary; - const float scale_factor = options_.gpu_scale_factor(); - if (image_frame_available_) { - const auto& input_frame = cc->Inputs().Tag(Tag).Get(); - width_ = RoundUp(input_frame.width(), alignment); - height_ = RoundUp(input_frame.height(), alignment); - } else { - width_ = RoundUp(options_.canvas_width_px(), alignment); - height_ = RoundUp(options_.canvas_height_px(), alignment); - } - width_canvas_ = RoundUp(width_ * scale_factor, alignment); - height_canvas_ = RoundUp(height_ * scale_factor, alignment); - - // Init texture for opencv rendered frame. - { - glGenTextures(1, &image_mat_tex_); - glBindTexture(GL_TEXTURE_2D, image_mat_tex_); - // TODO - // OpenCV only renders to RGB images, not RGBA. Ideally this should be RGBA. - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, width_canvas_, height_canvas_, 0, - GL_RGB, GL_UNSIGNED_BYTE, nullptr); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glBindTexture(GL_TEXTURE_2D, 0); - } -#endif // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.proto b/mediapipe/calculators/util/annotation_overlay_calculator.proto deleted file mode 100644 index 339bb2183..000000000 --- a/mediapipe/calculators/util/annotation_overlay_calculator.proto +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -// Options for the AnnotationOverlayCalculator. -message AnnotationOverlayCalculatorOptions { - extend CalculatorOptions { - optional AnnotationOverlayCalculatorOptions ext = 250607623; - } - - // The canvas width and height in pixels, and the background color. These - // options are used only if an input stream of ImageFrame isn't provided to - // the renderer calculator. If an input stream of ImageFrame is provided, then - // the calculator renders the annotations on top of the provided image, else a - // canvas is created with the dimensions and background color specified in - // these options and the annotations are rendered on top of this canvas. - optional int32 canvas_width_px = 2 [default = 1920]; - optional int32 canvas_height_px = 3 [default = 1080]; - optional Color canvas_color = 4; - - // Whether text should be rendered upside down. When it's set to false, text - // is rendered normally assuming the underlying image has its origin at the - // top-left corner. Therefore, for images with the origin at the bottom-left - // corner this should be set to true. - optional bool flip_text_vertically = 5 [default = false]; - - // Whether input stream IMAGE_GPU (OpenGL texture) has bottom-left or top-left - // origin. (Historically, OpenGL uses bottom left origin, but most MediaPipe - // examples expect textures to have top-left origin.) - optional bool gpu_uses_top_left_origin = 6 [default = true]; - - // Scale factor for intermediate image for GPU rendering. - // This can be used to speed up annotation by drawing the annotation on an - // intermediate image with a reduced scale, e.g. 0.5 (of the input image width - // and height), before resizing and overlaying it on top of the input image. - optional float gpu_scale_factor = 7 [default = 1.0]; -} diff --git a/mediapipe/calculators/util/association_calculator.h b/mediapipe/calculators/util/association_calculator.h deleted file mode 100644 index 6e5b480ce..000000000 --- a/mediapipe/calculators/util/association_calculator.h +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ - -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/util/association_calculator.pb.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/rectangle.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Computes the overlap similarity based on Intersection over Union (IoU) of -// two rectangles. -inline float OverlapSimilarity(const Rectangle_f& rect1, - const Rectangle_f& rect2) { - if (!rect1.Intersects(rect2)) return 0.0f; - // Compute IoU similarity score. - const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area(); - const float normalization = rect1.Area() + rect2.Area() - intersection_area; - return normalization > 0.0f ? intersection_area / normalization : 0.0f; -} - -// AssocationCalculator accepts multiple inputs of vectors of type T that can -// be converted to Rectangle_f. The output is a vector of type T that contains -// elements from the input vectors that don't overlap with each other. When -// two elements overlap, the element that comes in from a later input stream -// is kept in the output. This association operation is useful for multiple -// instance inference pipelines in MediaPipe. -// If an input stream is tagged with "PREV" tag, IDs of overlapping elements -// from "PREV" input stream are propagated to the output. Elements in the "PREV" -// input stream that don't overlap with other elements are not added to the -// output. This stream is designed to take detections from previous timestamp, -// e.g. output of PreviousLoopbackCalculator to provide temporal association. -// See AssociationDetectionCalculator and AssociationNormRectCalculator for -// example uses. -template -class AssociationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - // Atmost one input stream can be tagged with "PREV". - RET_CHECK_LE(cc->Inputs().NumEntries("PREV"), 1); - - if (cc->Inputs().HasTag("PREV")) { - RET_CHECK_GE(cc->Inputs().NumEntries(), 2); - } - - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - cc->Inputs().Get(id).Set>(); - } - - cc->Outputs().Index(0).Set>(); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - has_prev_input_stream_ = cc->Inputs().HasTag("PREV"); - if (has_prev_input_stream_) { - prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0); - } - options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>(); - CHECK_GE(options_.min_similarity_threshold(), 0); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - auto get_non_overlapping_elements = GetNonOverlappingElements(cc); - if (!get_non_overlapping_elements.ok()) { - return get_non_overlapping_elements.status(); - } - std::list result = get_non_overlapping_elements.value(); - - if (has_prev_input_stream_ && - !cc->Inputs().Get(prev_input_stream_id_).IsEmpty()) { - // Processed all regular input streams. Now compare the result list - // elements with those in the PREV input stream, and propagate IDs from - // PREV input stream as appropriate. - const std::vector& prev_input_vec = - cc->Inputs() - .Get(prev_input_stream_id_) - .template Get>(); - - MP_RETURN_IF_ERROR( - PropagateIdsFromPreviousToCurrent(prev_input_vec, &result)); - } - - auto output = absl::make_unique>(); - for (auto it = result.begin(); it != result.end(); ++it) { - output->push_back(*it); - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - - return absl::OkStatus(); - } - - protected: - ::mediapipe::AssociationCalculatorOptions options_; - - bool has_prev_input_stream_; - CollectionItemId prev_input_stream_id_; - - virtual absl::StatusOr GetRectangle(const T& input) { - return absl::OkStatus(); - } - - virtual std::pair GetId(const T& input) { return {false, -1}; } - - virtual void SetId(T* input, int id) {} - - private: - // Get a list of non-overlapping elements from all input streams, with - // increasing order of priority based on input stream index. - absl::StatusOr> GetNonOverlappingElements( - CalculatorContext* cc) { - std::list result; - - // Initialize result with the first non-empty input vector. - CollectionItemId non_empty_id = cc->Inputs().BeginId(); - for (CollectionItemId id = cc->Inputs().BeginId(); - id < cc->Inputs().EndId(); ++id) { - if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) { - continue; - } - const std::vector& input_vec = - cc->Inputs().Get(id).Get>(); - if (!input_vec.empty()) { - non_empty_id = id; - result.push_back(input_vec[0]); - for (int j = 1; j < input_vec.size(); ++j) { - MP_RETURN_IF_ERROR(AddElementToList(input_vec[j], &result)); - } - break; - } - } - - // Compare remaining input vectors with the non-empty result vector, - // remove lower-priority overlapping elements from the result vector and - // had corresponding higher-priority elements as necessary. - for (CollectionItemId id = non_empty_id + 1; id < cc->Inputs().EndId(); - ++id) { - if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) { - continue; - } - const std::vector& input_vec = - cc->Inputs().Get(id).Get>(); - - for (int vi = 0; vi < input_vec.size(); ++vi) { - MP_RETURN_IF_ERROR(AddElementToList(input_vec[vi], &result)); - } - } - - return result; - } - - absl::Status AddElementToList(T element, std::list* current) { - // Compare this element with elements of the input collection. If this - // element has high overlap with elements of the collection, remove - // those elements from the collection and add this element. - ASSIGN_OR_RETURN(auto cur_rect, GetRectangle(element)); - - bool change_id = false; - int new_elem_id = -1; - - for (auto uit = current->begin(); uit != current->end();) { - ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit)); - if (OverlapSimilarity(cur_rect, prev_rect) > - options_.min_similarity_threshold()) { - std::pair prev_id = GetId(*uit); - // If prev_id.first is false when some element doesn't have an ID, - // change_id and new_elem_id will not be updated. - if (prev_id.first) { - change_id = prev_id.first; - new_elem_id = prev_id.second; - } - uit = current->erase(uit); - } else { - ++uit; - } - } - - if (change_id) { - SetId(&element, new_elem_id); - } - current->push_back(element); - - return absl::OkStatus(); - } - - // Compare elements of the current list with elements in from the collection - // of elements from the previous input stream, and propagate IDs from the - // previous input stream as appropriate. - absl::Status PropagateIdsFromPreviousToCurrent( - const std::vector& prev_input_vec, std::list* current) { - for (auto vit = current->begin(); vit != current->end(); ++vit) { - auto get_cur_rectangle = GetRectangle(*vit); - if (!get_cur_rectangle.ok()) { - return get_cur_rectangle.status(); - } - const Rectangle_f& cur_rect = get_cur_rectangle.value(); - - bool change_id = false; - int id_for_vi = -1; - - for (int ui = 0; ui < prev_input_vec.size(); ++ui) { - auto get_prev_rectangle = GetRectangle(prev_input_vec[ui]); - if (!get_prev_rectangle.ok()) { - return get_prev_rectangle.status(); - } - const Rectangle_f& prev_rect = get_prev_rectangle.value(); - - if (OverlapSimilarity(cur_rect, prev_rect) > - options_.min_similarity_threshold()) { - std::pair prev_id = GetId(prev_input_vec[ui]); - // If prev_id.first is false when some element doesn't have an ID, - // change_id and id_for_vi will not be updated. - if (prev_id.first) { - change_id = prev_id.first; - id_for_vi = prev_id.second; - } - } - } - - if (change_id) { - T element = *vit; - SetId(&element, id_for_vi); - *vit = element; - } - } - return absl::OkStatus(); - } -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/association_calculator.proto b/mediapipe/calculators/util/association_calculator.proto deleted file mode 100644 index ca66f80b8..000000000 --- a/mediapipe/calculators/util/association_calculator.proto +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message AssociationCalculatorOptions { - extend CalculatorOptions { - optional AssociationCalculatorOptions ext = 275124847; - } - - optional float min_similarity_threshold = 1 [default = 1.0]; -} diff --git a/mediapipe/calculators/util/association_calculator_test.cc b/mediapipe/calculators/util/association_calculator_test.cc deleted file mode 100644 index 140338ba7..000000000 --- a/mediapipe/calculators/util/association_calculator_test.cc +++ /dev/null @@ -1,476 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -::mediapipe::Detection DetectionWithRelativeLocationData(double xmin, - double ymin, - double width, - double height) { - ::mediapipe::Detection detection; - ::mediapipe::LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(::mediapipe::LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(xmin); - location_data->mutable_relative_bounding_box()->set_ymin(ymin); - location_data->mutable_relative_bounding_box()->set_width(width); - location_data->mutable_relative_bounding_box()->set_height(height); - return detection; -} - -} // namespace - -class AssociationDetectionCalculatorTest : public ::testing::Test { - protected: - AssociationDetectionCalculatorTest() { - // 0.4 ================ - // | | | | - // 0.3 ===================== | DET2 | | - // | | | DET1 | | | DET4 | - // 0.2 | DET0 | =========== ================ - // | | | | | | - // 0.1 =====|=============== | - // | DET3 | | | - // 0.0 ================ | - // | DET5 | - // -0.1 =========== - // 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2 - - // Detection det_0. - det_0 = DetectionWithRelativeLocationData(/*xmin=*/0.1, /*ymin=*/0.1, - /*width=*/0.2, /*height=*/0.2); - det_0.set_detection_id(0); - - // Detection det_1. - det_1 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/0.1, - /*width=*/0.2, /*height=*/0.2); - det_1.set_detection_id(1); - - // Detection det_2. - det_2 = DetectionWithRelativeLocationData(/*xmin=*/0.9, /*ymin=*/0.2, - /*width=*/0.2, /*height=*/0.2); - det_2.set_detection_id(2); - - // Detection det_3. - det_3 = DetectionWithRelativeLocationData(/*xmin=*/0.2, /*ymin=*/0.0, - /*width=*/0.3, /*height=*/0.3); - det_3.set_detection_id(3); - - // Detection det_4. - det_4 = DetectionWithRelativeLocationData(/*xmin=*/1.0, /*ymin=*/0.2, - /*width=*/0.2, /*height=*/0.2); - det_4.set_detection_id(4); - - // Detection det_5. - det_5 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/-0.1, - /*width=*/0.3, /*height=*/0.3); - det_5.set_detection_id(5); - } - - ::mediapipe::Detection det_0, det_1, det_2, det_3, det_4, det_5; -}; - -TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationDetectionCalculator" - input_stream: "input_vec_0" - input_stream: "input_vec_1" - input_stream: "input_vec_2" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream 0: det_0, det_1, det_2. - auto input_vec_0 = absl::make_unique>(); - input_vec_0->push_back(det_0); - input_vec_0->push_back(det_1); - input_vec_0->push_back(det_2); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(input_vec_0.release()).At(Timestamp(1))); - - // Input Stream 1: det_3, det_4. - auto input_vec_1 = absl::make_unique>(); - input_vec_1->push_back(det_3); - input_vec_1->push_back(det_4); - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(input_vec_1.release()).At(Timestamp(1))); - - // Input Stream 2: det_5. - auto input_vec_2 = absl::make_unique>(); - input_vec_2->push_back(det_5); - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(input_vec_2.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // det_3 overlaps with det_0, det_1 and det_5 overlaps with det_3. Since det_5 - // is in the highest priority, we remove other rects. det_4 overlaps with - // det_2, and det_4 is higher priority, so we keep it. The final output - // therefore contains 2 elements. - EXPECT_EQ(2, assoc_rects.size()); - // Outputs are in order of inputs, so det_4 is before det_5 in output vector. - - // det_4 overlaps with det_2, so new id for det_4 is 2. - EXPECT_TRUE(assoc_rects[0].has_detection_id()); - EXPECT_EQ(2, assoc_rects[0].detection_id()); - det_4.set_detection_id(2); - EXPECT_THAT(assoc_rects[0], EqualsProto(det_4)); - - // det_3 overlaps with det_0, so new id for det_3 is 0. - // det_3 overlaps with det_1, so new id for det_3 is 1. - // det_5 overlaps with det_3, so new id for det_5 is 1. - EXPECT_TRUE(assoc_rects[1].has_detection_id()); - EXPECT_EQ(1, assoc_rects[1].detection_id()); - det_5.set_detection_id(1); - EXPECT_THAT(assoc_rects[1], EqualsProto(det_5)); -} - -TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationDetectionCalculator" - input_stream: "PREV:input_vec_0" - input_stream: "input_vec_1" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream 0: det_3, det_4. - auto input_vec_0 = absl::make_unique>(); - input_vec_0->push_back(det_3); - input_vec_0->push_back(det_4); - CollectionItemId prev_input_stream_id = - runner.MutableInputs()->GetId("PREV", 0); - runner.MutableInputs() - ->Get(prev_input_stream_id) - .packets.push_back(Adopt(input_vec_0.release()).At(Timestamp(1))); - - // Input Stream 1: det_5. - auto input_vec_1 = absl::make_unique>(); - input_vec_1->push_back(det_5); - CollectionItemId input_stream_id = runner.MutableInputs()->GetId("", 0); - runner.MutableInputs() - ->Get(input_stream_id) - .packets.push_back(Adopt(input_vec_1.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // det_5 overlaps with det_3 and doesn't overlap with det_4. Since det_4 is - // in the PREV input stream, it doesn't get copied to the output, so the final - // output contains 1 element. - EXPECT_EQ(1, assoc_rects.size()); - - // det_5 overlaps with det_3, det_3 is in PREV, so new id for det_5 is 3. - EXPECT_TRUE(assoc_rects[0].has_detection_id()); - EXPECT_EQ(3, assoc_rects[0].detection_id()); - det_5.set_detection_id(3); - EXPECT_THAT(assoc_rects[0], EqualsProto(det_5)); -} - -TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationDetectionCalculator" - input_stream: "input_vec_0" - input_stream: "input_vec_1" - input_stream: "input_vec_2" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream 0: det_5. - auto input_vec_0 = absl::make_unique>(); - input_vec_0->push_back(det_5); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(input_vec_0.release()).At(Timestamp(1))); - - // Input Stream 1: det_3, det_4. - auto input_vec_1 = absl::make_unique>(); - input_vec_1->push_back(det_3); - input_vec_1->push_back(det_4); - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(input_vec_1.release()).At(Timestamp(1))); - - // Input Stream 2: det_0, det_1, det_2. - auto input_vec_2 = absl::make_unique>(); - input_vec_2->push_back(det_0); - input_vec_2->push_back(det_1); - input_vec_2->push_back(det_2); - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(input_vec_2.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // det_3 overlaps with det_5, so det_5 is removed. det_0 overlaps with det_3, - // so det_3 is removed as det_0 is in higher priority for keeping. det_2 - // overlaps with det_4 so det_4 is removed as det_2 is higher priority for - // keeping. The final output therefore contains 3 elements. - EXPECT_EQ(3, assoc_rects.size()); - // Outputs are in same order as inputs. - - // det_3 overlaps with det_5, so new id for det_3 is 5. - // det_0 overlaps with det_3, so new id for det_0 is 5. - EXPECT_TRUE(assoc_rects[0].has_detection_id()); - EXPECT_EQ(5, assoc_rects[0].detection_id()); - det_0.set_detection_id(5); - EXPECT_THAT(assoc_rects[0], EqualsProto(det_0)); - - // det_1 stays with id 1. - EXPECT_TRUE(assoc_rects[1].has_detection_id()); - EXPECT_EQ(1, assoc_rects[1].detection_id()); - EXPECT_THAT(assoc_rects[1], EqualsProto(det_1)); - - // det_2 overlaps with det_4, so new id for det_2 is 4. - EXPECT_TRUE(assoc_rects[2].has_detection_id()); - EXPECT_EQ(4, assoc_rects[2].detection_id()); - det_2.set_detection_id(4); - EXPECT_THAT(assoc_rects[2], EqualsProto(det_2)); -} - -class AssociationNormRectCalculatorTest : public ::testing::Test { - protected: - AssociationNormRectCalculatorTest() { - // 0.4 ================ - // | | | | - // 0.3 ===================== | NR2 | | - // | | | NR1 | | | NR4 | - // 0.2 | NR0 | =========== ================ - // | | | | | | - // 0.1 =====|=============== | - // | NR3 | | | - // 0.0 ================ | - // | NR5 | - // -0.1 =========== - // 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 1.1 1.2 - - // NormalizedRect nr_0. - nr_0.set_x_center(0.2); - nr_0.set_y_center(0.2); - nr_0.set_width(0.2); - nr_0.set_height(0.2); - - // NormalizedRect nr_1. - nr_1.set_x_center(0.4); - nr_1.set_y_center(0.2); - nr_1.set_width(0.2); - nr_1.set_height(0.2); - - // NormalizedRect nr_2. - nr_2.set_x_center(1.0); - nr_2.set_y_center(0.3); - nr_2.set_width(0.2); - nr_2.set_height(0.2); - - // NormalizedRect nr_3. - nr_3.set_x_center(0.35); - nr_3.set_y_center(0.15); - nr_3.set_width(0.3); - nr_3.set_height(0.3); - - // NormalizedRect nr_4. - nr_4.set_x_center(1.1); - nr_4.set_y_center(0.3); - nr_4.set_width(0.2); - nr_4.set_height(0.2); - - // NormalizedRect nr_5. - nr_5.set_x_center(0.45); - nr_5.set_y_center(0.05); - nr_5.set_width(0.3); - nr_5.set_height(0.3); - } - - ::mediapipe::NormalizedRect nr_0, nr_1, nr_2, nr_3, nr_4, nr_5; -}; - -TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationNormRectCalculator" - input_stream: "input_vec_0" - input_stream: "input_vec_1" - input_stream: "input_vec_2" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream 0: nr_0, nr_1, nr_2. - auto input_vec_0 = - absl::make_unique>(); - input_vec_0->push_back(nr_0); - input_vec_0->push_back(nr_1); - input_vec_0->push_back(nr_2); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(input_vec_0.release()).At(Timestamp(1))); - - // Input Stream 1: nr_3, nr_4. - auto input_vec_1 = - absl::make_unique>(); - input_vec_1->push_back(nr_3); - input_vec_1->push_back(nr_4); - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(input_vec_1.release()).At(Timestamp(1))); - - // Input Stream 2: nr_5. - auto input_vec_2 = - absl::make_unique>(); - input_vec_2->push_back(nr_5); - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(input_vec_2.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // nr_3 overlaps with nr_0, nr_1 and nr_5 overlaps with nr_3. Since nr_5 is - // in the highest priority, we remove other rects. - // nr_4 overlaps with nr_2, and nr_4 is higher priority, so we keep it. - // The final output therefore contains 2 elements. - EXPECT_EQ(2, assoc_rects.size()); - // Outputs are in order of inputs, so nr_4 is before nr_5 in output vector. - EXPECT_THAT(assoc_rects[0], EqualsProto(nr_4)); - EXPECT_THAT(assoc_rects[1], EqualsProto(nr_5)); -} - -TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationNormRectCalculator" - input_stream: "input_vec_0" - input_stream: "input_vec_1" - input_stream: "input_vec_2" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream 0: nr_5. - auto input_vec_0 = - absl::make_unique>(); - input_vec_0->push_back(nr_5); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(input_vec_0.release()).At(Timestamp(1))); - - // Input Stream 1: nr_3, nr_4. - auto input_vec_1 = - absl::make_unique>(); - input_vec_1->push_back(nr_3); - input_vec_1->push_back(nr_4); - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(input_vec_1.release()).At(Timestamp(1))); - - // Input Stream 2: nr_0, nr_1, nr_2. - auto input_vec_2 = - absl::make_unique>(); - input_vec_2->push_back(nr_0); - input_vec_2->push_back(nr_1); - input_vec_2->push_back(nr_2); - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(input_vec_2.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // nr_3 overlaps with nr_5, so nr_5 is removed. nr_0 overlaps with nr_3, so - // nr_3 is removed as nr_0 is in higher priority for keeping. nr_2 overlaps - // with nr_4 so nr_4 is removed as nr_2 is higher priority for keeping. - // The final output therefore contains 3 elements. - EXPECT_EQ(3, assoc_rects.size()); - // Outputs are in same order as inputs. - EXPECT_THAT(assoc_rects[0], EqualsProto(nr_0)); - EXPECT_THAT(assoc_rects[1], EqualsProto(nr_1)); - EXPECT_THAT(assoc_rects[2], EqualsProto(nr_2)); -} - -TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "AssociationNormRectCalculator" - input_stream: "input_vec" - output_stream: "output_vec" - options { - [mediapipe.AssociationCalculatorOptions.ext] { - min_similarity_threshold: 0.1 - } - } - )pb")); - - // Input Stream : nr_3, nr_5. - auto input_vec = - absl::make_unique>(); - input_vec->push_back(nr_3); - input_vec->push_back(nr_5); - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(input_vec.release()).At(Timestamp(1))); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Index(0).packets; - EXPECT_EQ(1, output.size()); - const auto& assoc_rects = - output[0].Get>(); - - // nr_5 overlaps with nr_3. Since nr_5 is after nr_3 in the same input stream - // we remove nr_3 and keep nr_5. - // The final output therefore contains 1 elements. - EXPECT_EQ(1, assoc_rects.size()); - EXPECT_THAT(assoc_rects[0], EqualsProto(nr_5)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/association_detection_calculator.cc b/mediapipe/calculators/util/association_detection_calculator.cc deleted file mode 100644 index 35112aee7..000000000 --- a/mediapipe/calculators/util/association_detection_calculator.cc +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/association_calculator.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/rectangle.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// A subclass of AssociationCalculator for Detection. Example: -// node { -// calculator: "AssociationDetectionCalculator" -// input_stream: "PREV:input_vec_0" -// input_stream: "input_vec_1" -// input_stream: "input_vec_2" -// output_stream: "output_vec" -// options { -// [mediapipe.AssociationCalculatorOptions.ext] { -// min_similarity_threshold: 0.1 -// } -// } -class AssociationDetectionCalculator - : public AssociationCalculator<::mediapipe::Detection> { - public: - static absl::Status GetContract(CalculatorContract* cc) { - return AssociationCalculator<::mediapipe::Detection>::GetContract(cc); - } - - absl::Status Open(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::Detection>::Open(cc); - } - - absl::Status Process(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::Detection>::Process(cc); - } - - absl::Status Close(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::Detection>::Close(cc); - } - - protected: - absl::StatusOr GetRectangle( - const ::mediapipe::Detection& input) override { - if (!input.has_location_data()) { - return absl::InternalError("Missing location_data in Detection"); - } - const Location location(input.location_data()); - return location.GetRelativeBBox(); - } - - std::pair GetId(const ::mediapipe::Detection& input) override { - return {input.has_detection_id(), input.detection_id()}; - } - - void SetId(::mediapipe::Detection* input, int id) override { - input->set_detection_id(id); - } -}; - -REGISTER_CALCULATOR(AssociationDetectionCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/association_norm_rect_calculator.cc b/mediapipe/calculators/util/association_norm_rect_calculator.cc deleted file mode 100644 index a9194604a..000000000 --- a/mediapipe/calculators/util/association_norm_rect_calculator.cc +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/association_calculator.h" -#include "mediapipe/framework/calculator_context.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/rectangle.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// A subclass of AssociationCalculator for NormalizedRect. Example use case: -// node { -// calculator: "AssociationNormRectCalculator" -// input_stream: "input_vec_0" -// input_stream: "input_vec_1" -// input_stream: "input_vec_2" -// output_stream: "output_vec" -// options { -// [mediapipe.AssociationCalculatorOptions.ext] { -// min_similarity_threshold: 0.1 -// } -// } -class AssociationNormRectCalculator - : public AssociationCalculator<::mediapipe::NormalizedRect> { - public: - static absl::Status GetContract(CalculatorContract* cc) { - return AssociationCalculator<::mediapipe::NormalizedRect>::GetContract(cc); - } - - absl::Status Open(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::NormalizedRect>::Open(cc); - } - - absl::Status Process(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::NormalizedRect>::Process(cc); - } - - absl::Status Close(CalculatorContext* cc) override { - return AssociationCalculator<::mediapipe::NormalizedRect>::Close(cc); - } - - protected: - absl::StatusOr GetRectangle( - const ::mediapipe::NormalizedRect& input) override { - if (!input.has_x_center() || !input.has_y_center() || !input.has_width() || - !input.has_height()) { - return absl::InternalError("Missing dimensions in NormalizedRect."); - } - const float xmin = input.x_center() - input.width() / 2.0; - const float ymin = input.y_center() - input.height() / 2.0; - // TODO: Support rotation for rectangle. - return Rectangle_f(xmin, ymin, input.width(), input.height()); - } -}; - -REGISTER_CALCULATOR(AssociationNormRectCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/clock_latency_calculator.cc b/mediapipe/calculators/util/clock_latency_calculator.cc deleted file mode 100644 index 5c5711731..000000000 --- a/mediapipe/calculators/util/clock_latency_calculator.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/time/time.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace { -// Tag name for reference signal. -constexpr char kReferenceTag[] = "REFERENCE"; -} // namespace - -// A calculator that diffs multiple input absl::Time streams against a -// reference Time stream, and outputs the resulting absl::Duration's. Useful -// in combination with ClockTimestampCalculator to be able to determine the -// latency between two different points in a graph. -// -// Inputs: At least one non-reference Time stream is required. -// 0- Time stream 0 -// 1- Time stream 1 -// ... -// N- Time stream N -// REFERENCE_SIGNAL (required): The Time stream by which all others are -// compared. Should be the stream from which our other streams were -// computed, in order to provide meaningful latency results. -// -// Outputs: -// 0- Duration from REFERENCE_SIGNAL to input stream 0 -// 1- Duration from REFERENCE_SIGNAL to input stream 1 -// ... -// N- Duration from REFERENCE_SIGNAL to input stream N -// -// Example config: -// node { -// calculator: "ClockLatencyCalculator" -// input_stream: "packet_clocktime_stream_0" -// input_stream: "packet_clocktime_stream_1" -// input_stream: "packet_clocktime_stream_2" -// input_stream: "REFERENCE_SIGNAL: packet_clocktime_stream_reference" -// output_stream: "packet_latency_stream_0" -// output_stream: "packet_latency_stream_1" -// output_stream: "packet_latency_stream_2" -// } -// -class ClockLatencyCalculator : public CalculatorBase { - public: - ClockLatencyCalculator() {} - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - int64 num_packet_streams_ = -1; -}; -REGISTER_CALCULATOR(ClockLatencyCalculator); - -absl::Status ClockLatencyCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_GT(cc->Inputs().NumEntries(), 1); - - int64 num_packet_streams = cc->Inputs().NumEntries() - 1; - RET_CHECK_EQ(cc->Outputs().NumEntries(), num_packet_streams); - - for (int64 i = 0; i < num_packet_streams; ++i) { - cc->Inputs().Index(i).Set(); - cc->Outputs().Index(i).Set(); - } - cc->Inputs().Tag(kReferenceTag).Set(); - - return absl::OkStatus(); -} - -absl::Status ClockLatencyCalculator::Open(CalculatorContext* cc) { - // Direct passthrough, as far as timestamp and bounds are concerned. - cc->SetOffset(TimestampDiff(0)); - num_packet_streams_ = cc->Inputs().NumEntries() - 1; - return absl::OkStatus(); -} - -absl::Status ClockLatencyCalculator::Process(CalculatorContext* cc) { - // Get reference time. - RET_CHECK(!cc->Inputs().Tag(kReferenceTag).IsEmpty()); - const absl::Time& reference_time = - cc->Inputs().Tag(kReferenceTag).Get(); - - // Push Duration packets for every input stream we have. - for (int64 i = 0; i < num_packet_streams_; ++i) { - if (!cc->Inputs().Index(i).IsEmpty()) { - const absl::Time& input_stream_time = - cc->Inputs().Index(i).Get(); - cc->Outputs().Index(i).AddPacket( - MakePacket(input_stream_time - reference_time) - .At(cc->InputTimestamp())); - } - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/clock_timestamp_calculator.cc b/mediapipe/calculators/util/clock_timestamp_calculator.cc deleted file mode 100644 index 4ba56cfd0..000000000 --- a/mediapipe/calculators/util/clock_timestamp_calculator.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/time/time.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace { -// Tag name for clock side packet. -constexpr char kClockTag[] = "CLOCK"; -} // namespace - -// A calculator that outputs the current clock time at which it receives input -// packets. Use a separate instance of this calculator for each input stream -// you wish to output a clock time for. -// -// InputSidePacket (Optional): -// CLOCK: A clock to use for querying the current time. -// -// Inputs: -// A single packet stream we wish to get the current clocktime for - -// Outputs: -// A single stream of absl::Time packets, representing the clock time at which -// we received the input stream's packets. - -// Example config: -// node { -// calculator: "ClockTimestampCalculator" -// input_side_packet: "CLOCK:monotonic_clock" -// input_stream: "packet_stream" -// output_stream: "packet_clocktime_stream" -// } -// -class ClockTimestampCalculator : public CalculatorBase { - public: - ClockTimestampCalculator() {} - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Clock object. - std::shared_ptr<::mediapipe::Clock> clock_; -}; -REGISTER_CALCULATOR(ClockTimestampCalculator); - -absl::Status ClockTimestampCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Inputs().NumEntries(), 1); - RET_CHECK_EQ(cc->Outputs().NumEntries(), 1); - - cc->Inputs().Index(0).SetAny(); - cc->Outputs().Index(0).Set(); - - // Optional Clock input side packet. - if (cc->InputSidePackets().HasTag(kClockTag)) { - cc->InputSidePackets() - .Tag(kClockTag) - .Set>(); - } - - return absl::OkStatus(); -} - -absl::Status ClockTimestampCalculator::Open(CalculatorContext* cc) { - // Direct passthrough, as far as timestamp and bounds are concerned. - cc->SetOffset(TimestampDiff(0)); - - // Initialize the clock. - if (cc->InputSidePackets().HasTag(kClockTag)) { - clock_ = cc->InputSidePackets() - .Tag("CLOCK") - .Get>(); - } else { - clock_.reset( - ::mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock()); - } - - return absl::OkStatus(); -} - -absl::Status ClockTimestampCalculator::Process(CalculatorContext* cc) { - // Push the Time packet to output. - auto timestamp_packet = MakePacket(clock_->TimeNow()); - cc->Outputs().Index(0).AddPacket(timestamp_packet.At(cc->InputTimestamp())); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.cc b/mediapipe/calculators/util/collection_has_min_size_calculator.cc deleted file mode 100644 index 956818c87..000000000 --- a/mediapipe/calculators/util/collection_has_min_size_calculator.cc +++ /dev/null @@ -1,40 +0,0 @@ - -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/collection_has_min_size_calculator.h" - -#include - -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -typedef CollectionHasMinSizeCalculator> - NormalizedRectVectorHasMinSizeCalculator; -REGISTER_CALCULATOR(NormalizedRectVectorHasMinSizeCalculator); - -typedef CollectionHasMinSizeCalculator< - std::vector> - NormalizedLandmarkListVectorHasMinSizeCalculator; -REGISTER_CALCULATOR(NormalizedLandmarkListVectorHasMinSizeCalculator); - -typedef CollectionHasMinSizeCalculator< - std::vector> - ClassificationListVectorHasMinSizeCalculator; -REGISTER_CALCULATOR(ClassificationListVectorHasMinSizeCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.h b/mediapipe/calculators/util/collection_has_min_size_calculator.h deleted file mode 100644 index 4d4b6a678..000000000 --- a/mediapipe/calculators/util/collection_has_min_size_calculator.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ - -#include - -#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Deterimines if an input iterable collection has a minimum size, specified -// in CollectionHasMinSizeCalculatorOptions. Example usage: -// node { -// calculator: "IntVectorHasMinSizeCalculator" -// input_stream: "ITERABLE:input_int_vector" -// output_stream: "has_min_ints" -// options { -// [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { -// min_size: 2 -// } -// } -// } -// Optionally, uses a side packet to override `min_size` specified in the -// calculator options. -template -class CollectionHasMinSizeCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("ITERABLE")); - RET_CHECK_EQ(1, cc->Inputs().NumEntries()); - - RET_CHECK_EQ(1, cc->Outputs().NumEntries()); - - RET_CHECK_GE( - cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>() - .min_size(), - 0); - - cc->Inputs().Tag("ITERABLE").Set(); - cc->Outputs().Index(0).Set(); - - // Optional input side packet that determines `min_size_`. - if (cc->InputSidePackets().NumEntries() > 0) { - cc->InputSidePackets().Index(0).Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - min_size_ = - cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>() - .min_size(); - // Override `min_size` if passed as side packet. - if (cc->InputSidePackets().NumEntries() > 0 && - !cc->InputSidePackets().Index(0).IsEmpty()) { - min_size_ = cc->InputSidePackets().Index(0).Get(); - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const IterableT& input = cc->Inputs().Tag("ITERABLE").Get(); - bool has_min_size = input.size() >= min_size_; - - cc->Outputs().Index(0).AddPacket( - MakePacket(has_min_size).At(cc->InputTimestamp())); - - return absl::OkStatus(); - } - - private: - int min_size_ = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator.proto b/mediapipe/calculators/util/collection_has_min_size_calculator.proto deleted file mode 100644 index f482277c9..000000000 --- a/mediapipe/calculators/util/collection_has_min_size_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message CollectionHasMinSizeCalculatorOptions { - extend CalculatorOptions { - optional CollectionHasMinSizeCalculatorOptions ext = 259397840; - } - - // The minimum size an input iterable collection should have for the - // calculator to output true. - optional int32 min_size = 1 [default = 0]; -} diff --git a/mediapipe/calculators/util/collection_has_min_size_calculator_test.cc b/mediapipe/calculators/util/collection_has_min_size_calculator_test.cc deleted file mode 100644 index 805ad495d..000000000 --- a/mediapipe/calculators/util/collection_has_min_size_calculator_test.cc +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/collection_has_min_size_calculator.h" - -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" // NOLINT - -namespace mediapipe { - -typedef CollectionHasMinSizeCalculator> - TestIntCollectionHasMinSizeCalculator; -REGISTER_CALCULATOR(TestIntCollectionHasMinSizeCalculator); - -void AddInputVector(const std::vector& input, int64 timestamp, - CalculatorRunner* runner) { - runner->MutableInputs() - ->Tag("ITERABLE") - .packets.push_back( - MakePacket>(input).At(Timestamp(timestamp))); -} - -TEST(TestIntCollectionHasMinSizeCalculator, DoesHaveMinSize) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestIntCollectionHasMinSizeCalculator" - input_stream: "ITERABLE:input_vector" - output_stream: "output_vector" - options { - [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 2 } - } - )pb"); - CalculatorRunner runner(node_config); - const std::vector& outputs = runner.Outputs().Index(0).packets; - - AddInputVector({1, 2}, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - EXPECT_TRUE(outputs[0].Get()); - - AddInputVector({1, 2, 3}, /*timestamp=*/2, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(2, outputs.size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - EXPECT_TRUE(outputs[1].Get()); -} - -TEST(TestIntCollectionHasMinSizeCalculator, - DoesHaveMinSize_MinSizeAsSidePacket) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestIntCollectionHasMinSizeCalculator" - input_stream: "ITERABLE:input_vector" - input_side_packet: "min_size" - output_stream: "output_vector" - )pb"); - CalculatorRunner runner(node_config); - const std::vector& outputs = runner.Outputs().Index(0).packets; - - runner.MutableSidePackets()->Index(0) = MakePacket(2); - - AddInputVector({1, 2}, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - EXPECT_TRUE(outputs[0].Get()); - - AddInputVector({1, 2, 3}, /*timestamp=*/2, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(2, outputs.size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - EXPECT_TRUE(outputs[1].Get()); -} - -TEST(TestIntCollectionHasMinSizeCalculator, DoesNotHaveMinSize) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestIntCollectionHasMinSizeCalculator" - input_stream: "ITERABLE:input_vector" - output_stream: "output_vector" - options { - [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] { min_size: 3 } - } - )pb"); - CalculatorRunner runner(node_config); - const std::vector& outputs = runner.Outputs().Index(0).packets; - - AddInputVector({1}, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - EXPECT_FALSE(outputs[0].Get()); - - AddInputVector({1, 2}, /*timestamp=*/2, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(2, outputs.size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - EXPECT_FALSE(outputs[1].Get()); -} - -TEST(TestIntCollectionHasMinSizeCalculator, - DoesNotHaveMinSize_MinSizeAsSidePacket) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "TestIntCollectionHasMinSizeCalculator" - input_stream: "ITERABLE:input_vector" - input_side_packet: "min_size" - output_stream: "output_vector" - )pb"); - CalculatorRunner runner(node_config); - const std::vector& outputs = runner.Outputs().Index(0).packets; - - runner.MutableSidePackets()->Index(0) = MakePacket(3); - - AddInputVector({1}, /*timestamp=*/1, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(1, outputs.size()); - EXPECT_EQ(Timestamp(1), outputs[0].Timestamp()); - EXPECT_FALSE(outputs[0].Get()); - - AddInputVector({1, 2}, /*timestamp=*/2, &runner); - MP_ASSERT_OK(runner.Run()); - - EXPECT_EQ(2, outputs.size()); - EXPECT_EQ(Timestamp(2), outputs[1].Timestamp()); - EXPECT_FALSE(outputs[1].Get()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_classifications_merger_calculator.cc b/mediapipe/calculators/util/detection_classifications_merger_calculator.cc deleted file mode 100644 index 86f26b0dc..000000000 --- a/mediapipe/calculators/util/detection_classifications_merger_calculator.cc +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/substitute.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { -namespace api2 { - -namespace {} // namespace - -// Replaces the classification labels and scores from the input `Detection` with -// the ones provided into the input `ClassificationList`. Namely: -// * `label_id[i]` becomes `classification[i].index` -// * `score[i]` becomes `classification[i].score` -// * `label[i]` becomes `classification[i].label` (if present) -// -// In case the input `ClassificationList` contains no results (i.e. -// `classification` is empty, which may happen if the classifier uses a score -// threshold and no confident enough result were returned), the input -// `Detection` is returned unchanged. -// -// This is specifically designed for two-stage detection cascades where the -// detections returned by a standalone detector (typically a class-agnostic -// localizer) are fed e.g. into a `TfLiteTaskImageClassifierCalculator` through -// the optional "RECT" or "NORM_RECT" input, e.g: -// -// node { -// calculator: "DetectionsToRectsCalculator" -// # Output of an upstream object detector. -// input_stream: "DETECTION:detection" -// output_stream: "NORM_RECT:norm_rect" -// } -// node { -// calculator: "TfLiteTaskImageClassifierCalculator" -// input_stream: "IMAGE:image" -// input_stream: "NORM_RECT:norm_rect" -// output_stream: "CLASSIFICATION_RESULT:classification_result" -// } -// node { -// calculator: "TfLiteTaskClassificationResultToClassificationsCalculator" -// input_stream: "CLASSIFICATION_RESULT:classification_result" -// output_stream: "CLASSIFICATION_LIST:classification_list" -// } -// node { -// calculator: "DetectionClassificationsMergerCalculator" -// input_stream: "INPUT_DETECTION:detection" -// input_stream: "CLASSIFICATION_LIST:classification_list" -// # Final output. -// output_stream: "OUTPUT_DETECTION:classified_detection" -// } -// -// Inputs: -// INPUT_DETECTION: `Detection` proto. -// CLASSIFICATION_LIST: `ClassificationList` proto. -// -// Output: -// OUTPUT_DETECTION: modified `Detection` proto. -class DetectionClassificationsMergerCalculator : public Node { - public: - static constexpr Input kInputDetection{"INPUT_DETECTION"}; - static constexpr Input kClassificationList{ - "CLASSIFICATION_LIST"}; - static constexpr Output kOutputDetection{"OUTPUT_DETECTION"}; - - MEDIAPIPE_NODE_CONTRACT(kInputDetection, kClassificationList, - kOutputDetection); - - absl::Status Process(CalculatorContext* cc) override; -}; -MEDIAPIPE_REGISTER_NODE(DetectionClassificationsMergerCalculator); - -absl::Status DetectionClassificationsMergerCalculator::Process( - CalculatorContext* cc) { - if (kInputDetection(cc).IsEmpty() && kClassificationList(cc).IsEmpty()) { - return absl::OkStatus(); - } - RET_CHECK(!kInputDetection(cc).IsEmpty()); - RET_CHECK(!kClassificationList(cc).IsEmpty()); - - Detection detection = *kInputDetection(cc); - const ClassificationList& classification_list = *kClassificationList(cc); - - // Update input detection only if classification did return results. - if (classification_list.classification_size() != 0) { - detection.clear_label_id(); - detection.clear_score(); - detection.clear_label(); - detection.clear_display_name(); - for (const auto& classification : classification_list.classification()) { - if (!classification.has_index()) { - return absl::InvalidArgumentError( - "Missing required 'index' field in Classification proto."); - } - detection.add_label_id(classification.index()); - if (!classification.has_score()) { - return absl::InvalidArgumentError( - "Missing required 'score' field in Classification proto."); - } - detection.add_score(classification.score()); - if (classification.has_label()) { - detection.add_label(classification.label()); - } - if (classification.has_display_name()) { - detection.add_display_name(classification.display_name()); - } - } - // Post-conversion sanity checks. - if (detection.label_size() != 0 && - detection.label_size() != detection.label_id_size()) { - return absl::InvalidArgumentError(absl::Substitute( - "Each input Classification is expected to either always or never " - "provide a 'label' field. Found $0 'label' fields for $1 " - "'Classification' objects.", - /*$0=*/detection.label_size(), /*$1=*/detection.label_id_size())); - } - if (detection.display_name_size() != 0 && - detection.display_name_size() != detection.label_id_size()) { - return absl::InvalidArgumentError(absl::Substitute( - "Each input Classification is expected to either always or never " - "provide a 'display_name' field. Found $0 'display_name' fields for " - "$1 'Classification' objects.", - /*$0=*/detection.display_name_size(), - /*$1=*/detection.label_id_size())); - } - } - kOutputDetection(cc).Send(detection); - return absl::OkStatus(); -} - -} // namespace api2 -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc b/mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc deleted file mode 100644 index 56b1decdb..000000000 --- a/mediapipe/calculators/util/detection_classifications_merger_calculator_test.cc +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -constexpr char kGraphConfig[] = R"( - input_stream: "input_detection" - input_stream: "classification_list" - output_stream: "output_detection" - node { - calculator: "DetectionClassificationsMergerCalculator" - input_stream: "INPUT_DETECTION:input_detection" - input_stream: "CLASSIFICATION_LIST:classification_list" - output_stream: "OUTPUT_DETECTION:output_detection" - } - )"; - -constexpr char kInputDetection[] = R"( - label: "entity" - label_id: 1 - score: 0.9 - location_data { - format: BOUNDING_BOX - bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 } - } - display_name: "Entity" - )"; - -// Checks that the input Detection is returned unchanged if the input -// ClassificationList does not contain any result. -TEST(DetectionClassificationsMergerCalculator, SucceedsWithNoClassification) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(""); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and validate output. - EXPECT_THAT(output_packets, testing::SizeIs(1)); - const Detection& output_detection = output_packets[0].Get(); - EXPECT_THAT(output_detection, mediapipe::EqualsProto(input_detection)); -} - -// Checks that merging succeeds when the input ClassificationList includes -// labels and display names. -TEST(DetectionClassificationsMergerCalculator, - SucceedsWithLabelsAndDisplayNames) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" } - classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and validate output. - EXPECT_THAT(output_packets, testing::SizeIs(1)); - const Detection& output_detection = output_packets[0].Get(); - EXPECT_THAT(output_detection, - mediapipe::EqualsProto(ParseTextProtoOrDie(R"pb( - label: "dog" - label: "fox" - label_id: 11 - label_id: 12 - score: 0.5 - score: 0.4 - location_data { - format: BOUNDING_BOX - bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 } - } - display_name: "Dog" - display_name: "Fox" - )pb"))); -} - -// Checks that merging succeeds when the input ClassificationList doesn't -// include labels and display names. -TEST(DetectionClassificationsMergerCalculator, - SucceedsWithoutLabelsAndDisplayNames) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { index: 11 score: 0.5 } - classification { index: 12 score: 0.4 } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - MP_ASSERT_OK(graph.WaitUntilIdle()); - - // Get and validate output. - EXPECT_THAT(output_packets, testing::SizeIs(1)); - const Detection& output_detection = output_packets[0].Get(); - EXPECT_THAT(output_detection, - mediapipe::EqualsProto(ParseTextProtoOrDie(R"pb( - label_id: 11 - label_id: 12 - score: 0.5 - score: 0.4 - location_data { - format: BOUNDING_BOX - bounding_box { xmin: 50 ymin: 60 width: 70 height: 80 } - } - )pb"))); -} - -// Checks that merging fails if the input ClassificationList misses mandatory -// "index" field. -TEST(DetectionClassificationsMergerCalculator, FailsWithMissingIndex) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { score: 0.5 label: "dog" } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument); -} - -// Checks that merging fails if the input ClassificationList misses mandatory -// "score" field. -TEST(DetectionClassificationsMergerCalculator, FailsWithMissingScore) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { index: 11 label: "dog" } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument); -} - -// Checks that merging fails if the input ClassificationList has an -// inconsistent number of labels. -TEST(DetectionClassificationsMergerCalculator, - FailsWithInconsistentNumberOfLabels) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { index: 11 score: 0.5 label: "dog" display_name: "Dog" } - classification { index: 12 score: 0.4 display_name: "Fox" } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument); -} - -// Checks that merging fails if the input ClassificationList has an -// inconsistent number of display names. -TEST(DetectionClassificationsMergerCalculator, - FailsWithInconsistentNumberOfDisplayNames) { - auto graph_config = ParseTextProtoOrDie(kGraphConfig); - - // Prepare input packets. - const Detection& input_detection = - ParseTextProtoOrDie(kInputDetection); - Packet input_detection_packet = - MakePacket(input_detection).At(Timestamp(0)); - const ClassificationList& classification_list = - ParseTextProtoOrDie(R"pb( - classification { index: 11 score: 0.5 label: "dog" } - classification { index: 12 score: 0.4 label: "fox" display_name: "Fox" } - )pb"); - Packet classification_list_packet = - MakePacket(classification_list).At(Timestamp(0)); - - // Catch output. - std::vector output_packets; - tool::AddVectorSink("output_detection", &graph_config, &output_packets); - - // Run the graph. - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(graph_config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK( - graph.AddPacketToInputStream("input_detection", input_detection_packet)); - MP_ASSERT_OK(graph.AddPacketToInputStream("classification_list", - classification_list_packet)); - ASSERT_EQ(graph.WaitUntilIdle().code(), absl::StatusCode::kInvalidArgument); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc b/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc deleted file mode 100644 index 07fe791f6..000000000 --- a/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/node_hash_map.h" -#include "mediapipe/calculators/util/detection_label_id_to_text_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/resource_util.h" - -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -// Takes a label map (from label IDs to names), and replaces the label IDs -// in Detection protos with label names. Note that the calculator makes a copy -// of the input detections. Consider using it only when the size of input -// detections is small. -// -// Example usage: -// node { -// calculator: "DetectionLabelIdToTextCalculator" -// input_stream: "input_detections" -// output_stream: "output_detections" -// node_options: { -// [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] { -// label_map_path: "labelmap.txt" -// } -// } -// } -class DetectionLabelIdToTextCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::node_hash_map label_map_; - ::mediapipe::DetectionLabelIdToTextCalculatorOptions options_; -}; -REGISTER_CALCULATOR(DetectionLabelIdToTextCalculator); - -absl::Status DetectionLabelIdToTextCalculator::GetContract( - CalculatorContract* cc) { - cc->Inputs().Index(0).Set>(); - cc->Outputs().Index(0).Set>(); - - return absl::OkStatus(); -} - -absl::Status DetectionLabelIdToTextCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = - cc->Options<::mediapipe::DetectionLabelIdToTextCalculatorOptions>(); - - if (options_.has_label_map_path()) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, - PathToResourceAsFile(options_.label_map_path())); - std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - } else { - for (int i = 0; i < options_.label_size(); ++i) { - label_map_[i] = options_.label(i); - } - } - return absl::OkStatus(); -} - -absl::Status DetectionLabelIdToTextCalculator::Process(CalculatorContext* cc) { - std::vector output_detections; - for (const auto& input_detection : - cc->Inputs().Index(0).Get>()) { - output_detections.push_back(input_detection); - Detection& output_detection = output_detections.back(); - bool has_text_label = false; - for (const int32 label_id : output_detection.label_id()) { - if (label_map_.find(label_id) != label_map_.end()) { - output_detection.add_label(label_map_[label_id]); - has_text_label = true; - } - } - // Remove label_id field if text labels exist. - if (has_text_label && !options_.keep_label_id()) { - output_detection.clear_label_id(); - } - } - cc->Outputs().Index(0).AddPacket( - MakePacket>(output_detections) - .At(cc->InputTimestamp())); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_label_id_to_text_calculator.proto b/mediapipe/calculators/util/detection_label_id_to_text_calculator.proto deleted file mode 100644 index 198ca4d65..000000000 --- a/mediapipe/calculators/util/detection_label_id_to_text_calculator.proto +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message DetectionLabelIdToTextCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional DetectionLabelIdToTextCalculatorOptions ext = 251889072; - } - - // Path to a label map file for getting the actual name of detected classes. - optional string label_map_path = 1; - - // Alternative way to specify label map - // label: "label for id 0" - // label: "label for id 1" - // ... - repeated string label = 2; - - // By default, the `label_id` field from the input is stripped if a text label - // could be found. By setting this field to true, it is always copied to the - // output detections. - optional bool keep_label_id = 3; -} diff --git a/mediapipe/calculators/util/detection_letterbox_removal_calculator.cc b/mediapipe/calculators/util/detection_letterbox_removal_calculator.cc deleted file mode 100644 index 8f8025576..000000000 --- a/mediapipe/calculators/util/detection_letterbox_removal_calculator.cc +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING"; - -} // namespace - -// Adjusts detection locations on a letterboxed image to the corresponding -// locations on the same image with the letterbox removed. This is useful to map -// the detections inferred from a letterboxed image, for example, output of -// the ImageTransformationCalculator when the scale mode is FIT, back to the -// corresponding input image before letterboxing. -// -// Input: -// DETECTIONS: An std::vector representing detections on an -// letterboxed image. -// -// LETTERBOX_PADDING: An std::array representing the letterbox -// padding from the 4 sides ([left, top, right, bottom]) of the letterboxed -// image, normalized to [0.f, 1.f] by the letterboxed image dimensions. -// -// Output: -// DETECTIONS: An std::vector representing detections with their -// locations adjusted to the letterbox-removed (non-padded) image. -// -// Usage example: -// node { -// calculator: "DetectionLetterboxRemovalCalculator" -// input_stream: "DETECTIONS:detections" -// input_stream: "LETTERBOX_PADDING:letterbox_padding" -// output_stream: "DETECTIONS:adjusted_detections" -// } -class DetectionLetterboxRemovalCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionsTag) && - cc->Inputs().HasTag(kLetterboxPaddingTag)) - << "Missing one or more input streams."; - - cc->Inputs().Tag(kDetectionsTag).Set>(); - cc->Inputs().Tag(kLetterboxPaddingTag).Set>(); - - cc->Outputs().Tag(kDetectionsTag).Set>(); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - // Only process if there's input detections. - if (cc->Inputs().Tag(kDetectionsTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& input_detections = - cc->Inputs().Tag(kDetectionsTag).Get>(); - const auto& letterbox_padding = - cc->Inputs().Tag(kLetterboxPaddingTag).Get>(); - - const float left = letterbox_padding[0]; - const float top = letterbox_padding[1]; - const float left_and_right = letterbox_padding[0] + letterbox_padding[2]; - const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3]; - - auto output_detections = absl::make_unique>(); - for (const auto& detection : input_detections) { - Detection new_detection; - new_detection.CopyFrom(detection); - LocationData::RelativeBoundingBox* relative_bbox = - new_detection.mutable_location_data() - ->mutable_relative_bounding_box(); - - relative_bbox->set_xmin( - (detection.location_data().relative_bounding_box().xmin() - left) / - (1.0f - left_and_right)); - relative_bbox->set_ymin( - (detection.location_data().relative_bounding_box().ymin() - top) / - (1.0f - top_and_bottom)); - // The size of the bounding box will change as well. - relative_bbox->set_width( - detection.location_data().relative_bounding_box().width() / - (1.0f - left_and_right)); - relative_bbox->set_height( - detection.location_data().relative_bounding_box().height() / - (1.0f - top_and_bottom)); - - // Adjust keypoints as well. - for (int i = 0; - i < new_detection.mutable_location_data()->relative_keypoints_size(); - ++i) { - auto* keypoint = - new_detection.mutable_location_data()->mutable_relative_keypoints( - i); - const float new_x = (keypoint->x() - left) / (1.0f - left_and_right); - const float new_y = (keypoint->y() - top) / (1.0f - top_and_bottom); - keypoint->set_x(new_x); - keypoint->set_y(new_y); - } - - output_detections->emplace_back(new_detection); - } - - cc->Outputs() - .Tag("DETECTIONS") - .Add(output_detections.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(DetectionLetterboxRemovalCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_letterbox_removal_calculator_test.cc b/mediapipe/calculators/util/detection_letterbox_removal_calculator_test.cc deleted file mode 100644 index 343ccea4f..000000000 --- a/mediapipe/calculators/util/detection_letterbox_removal_calculator_test.cc +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -LocationData CreateRelativeLocationData(double xmin, double ymin, double width, - double height) { - LocationData location_data; - location_data.set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data.mutable_relative_bounding_box()->set_xmin(xmin); - location_data.mutable_relative_bounding_box()->set_ymin(ymin); - location_data.mutable_relative_bounding_box()->set_width(width); - location_data.mutable_relative_bounding_box()->set_height(height); - return location_data; -} - -Detection CreateDetection(const std::vector& labels, - const std::vector& label_ids, - const std::vector& scores, - const LocationData& location_data, - const std::string& feature_tag) { - Detection detection; - for (const auto& label : labels) { - detection.add_label(label); - } - for (const auto& label_id : label_ids) { - detection.add_label_id(label_id); - } - for (const auto& score : scores) { - detection.add_score(score); - } - *(detection.mutable_location_data()) = location_data; - detection.set_feature_tag(feature_tag); - return detection; -} - -CalculatorGraphConfig::Node GetDefaultNode() { - return ParseTextProtoOrDie(R"pb( - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:adjusted_detections" - )pb"); -} - -TEST(DetectionLetterboxRemovalCalculatorTest, PaddingLeftRight) { - CalculatorRunner runner(GetDefaultNode()); - - LocationData location_data = - CreateRelativeLocationData(0.25f, 0.25f, 0.25f, 0.25f); - const std::string label = "detected_object"; - - auto detections = absl::make_unique>(); - detections->push_back( - CreateDetection({label}, {}, {0.3f}, location_data, "feature_tag")); - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - auto padding = absl::make_unique>( - std::array{0.2f, 0.f, 0.3f, 0.f}); - runner.MutableInputs() - ->Tag("LETTERBOX_PADDING") - .packets.push_back(Adopt(padding.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("DETECTIONS").packets; - ASSERT_EQ(1, output.size()); - const auto& output_detections = output[0].Get>(); - - EXPECT_EQ(output_detections.size(), 1); - const auto& output_detection = output_detections[0]; - - EXPECT_EQ(output_detection.label_size(), 1); - EXPECT_EQ(output_detection.label(0), label); - EXPECT_EQ(output_detection.label_id_size(), 0); - EXPECT_EQ(output_detection.score_size(), 1); - EXPECT_EQ(output_detection.score(0), 0.3f); - - EXPECT_EQ(output_detection.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().xmin(), - testing::FloatNear(0.1f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().ymin(), - testing::FloatNear(0.25f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().width(), - testing::FloatNear(0.5f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().height(), - testing::FloatNear(0.25f, 1e-5)); -} - -TEST(DetectionLetterboxRemovalCalculatorTest, PaddingTopBottom) { - CalculatorRunner runner(GetDefaultNode()); - - LocationData location_data = - CreateRelativeLocationData(0.25f, 0.25f, 0.25f, 0.25f); - const std::string label = "detected_object"; - - auto detections = absl::make_unique>(); - detections->push_back( - CreateDetection({label}, {}, {0.3f}, location_data, "feature_tag")); - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - auto padding = absl::make_unique>( - std::array{0.f, 0.2f, 0.f, 0.3f}); - runner.MutableInputs() - ->Tag("LETTERBOX_PADDING") - .packets.push_back(Adopt(padding.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("DETECTIONS").packets; - ASSERT_EQ(1, output.size()); - const auto& output_detections = output[0].Get>(); - - EXPECT_EQ(output_detections.size(), 1); - const auto& output_detection = output_detections[0]; - - EXPECT_EQ(output_detection.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().xmin(), - testing::FloatNear(0.25f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().ymin(), - testing::FloatNear(0.1f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().width(), - testing::FloatNear(0.25f, 1e-5)); - EXPECT_THAT(output_detection.location_data().relative_bounding_box().height(), - testing::FloatNear(0.5f, 1e-5)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_projection_calculator.cc b/mediapipe/calculators/util/detection_projection_calculator.cc deleted file mode 100644 index 211fd204c..000000000 --- a/mediapipe/calculators/util/detection_projection_calculator.cc +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/point2.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Projects detections to a different coordinate system using a provided -// projection matrix. -// -// Input: -// DETECTIONS - std::vector -// Detections to project using the provided projection matrix. -// PROJECTION_MATRIX - std::array -// A 4x4 row-major-order matrix that maps data from one coordinate system to -// another. -// -// Output: -// DETECTIONS - std::vector -// Projected detections. -// -// Example: -// node { -// calculator: "DetectionProjectionCalculator" -// input_stream: "DETECTIONS:detections" -// input_stream: "PROJECTION_MATRIX:matrix" -// output_stream: "DETECTIONS:projected_detections" -// } -class DetectionProjectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(DetectionProjectionCalculator); - -namespace { - -constexpr char kDetections[] = "DETECTIONS"; -constexpr char kProjectionMatrix[] = "PROJECTION_MATRIX"; - -absl::Status ProjectDetection( - const std::function& project_fn, - Detection* detection) { - auto* location_data = detection->mutable_location_data(); - RET_CHECK_EQ(location_data->format(), LocationData::RELATIVE_BOUNDING_BOX); - - // Project keypoints. - for (int i = 0; i < location_data->relative_keypoints_size(); ++i) { - auto* kp = location_data->mutable_relative_keypoints(i); - const auto point = project_fn({kp->x(), kp->y()}); - kp->set_x(point.x()); - kp->set_y(point.y()); - } - - // Project bounding box. - auto* box = location_data->mutable_relative_bounding_box(); - - const float xmin = box->xmin(); - const float ymin = box->ymin(); - const float width = box->width(); - const float height = box->height(); - // a) Define and project box points. - std::array box_coordinates = { - Point2_f{xmin, ymin}, Point2_f{xmin + width, ymin}, - Point2_f{xmin + width, ymin + height}, Point2_f{xmin, ymin + height}}; - std::transform(box_coordinates.begin(), box_coordinates.end(), - box_coordinates.begin(), project_fn); - // b) Find new left top and right bottom points for a box which encompases - // non-projected (rotated) box. - constexpr float kFloatMax = std::numeric_limits::max(); - constexpr float kFloatMin = std::numeric_limits::lowest(); - Point2_f left_top = {kFloatMax, kFloatMax}; - Point2_f right_bottom = {kFloatMin, kFloatMin}; - std::for_each(box_coordinates.begin(), box_coordinates.end(), - [&left_top, &right_bottom](const Point2_f& p) { - left_top.set_x(std::min(left_top.x(), p.x())); - left_top.set_y(std::min(left_top.y(), p.y())); - right_bottom.set_x(std::max(right_bottom.x(), p.x())); - right_bottom.set_y(std::max(right_bottom.y(), p.y())); - }); - box->set_xmin(left_top.x()); - box->set_ymin(left_top.y()); - box->set_width(right_bottom.x() - left_top.x()); - box->set_height(right_bottom.y() - left_top.y()); - - return absl::OkStatus(); -} - -} // namespace - -absl::Status DetectionProjectionCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetections) && - cc->Inputs().HasTag(kProjectionMatrix)) - << "Missing one or more input streams."; - - RET_CHECK_EQ(cc->Inputs().NumEntries(kDetections), - cc->Outputs().NumEntries(kDetections)) - << "Same number of DETECTIONS input and output is required."; - - for (CollectionItemId id = cc->Inputs().BeginId(kDetections); - id != cc->Inputs().EndId(kDetections); ++id) { - cc->Inputs().Get(id).Set>(); - } - cc->Inputs().Tag(kProjectionMatrix).Set>(); - - for (CollectionItemId id = cc->Outputs().BeginId(kDetections); - id != cc->Outputs().EndId(kDetections); ++id) { - cc->Outputs().Get(id).Set>(); - } - - return absl::OkStatus(); -} - -absl::Status DetectionProjectionCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); -} - -absl::Status DetectionProjectionCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().Tag(kProjectionMatrix).IsEmpty()) { - return absl::OkStatus(); - } - const auto& project_mat = - cc->Inputs().Tag(kProjectionMatrix).Get>(); - auto project_fn = [project_mat](const Point2_f& p) -> Point2_f { - return {p.x() * project_mat[0] + p.y() * project_mat[1] + project_mat[3], - p.x() * project_mat[4] + p.y() * project_mat[5] + project_mat[7]}; - }; - - CollectionItemId input_id = cc->Inputs().BeginId(kDetections); - CollectionItemId output_id = cc->Outputs().BeginId(kDetections); - // Number of inputs and outpus is the same according to the contract. - for (; input_id != cc->Inputs().EndId(kDetections); ++input_id, ++output_id) { - const auto& input_packet = cc->Inputs().Get(input_id); - if (input_packet.IsEmpty()) { - continue; - } - - std::vector output_detections; - for (const auto& detection : input_packet.Get>()) { - Detection output_detection = detection; - MP_RETURN_IF_ERROR(ProjectDetection(project_fn, &output_detection)); - output_detections.push_back(std::move(output_detection)); - } - - cc->Outputs().Get(output_id).AddPacket( - MakePacket>(std::move(output_detections)) - .At(cc->InputTimestamp())); - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_projection_calculator_test.cc b/mediapipe/calculators/util/detection_projection_calculator_test.cc deleted file mode 100644 index 4cc85acee..000000000 --- a/mediapipe/calculators/util/detection_projection_calculator_test.cc +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/point2.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -using ::testing::ElementsAre; -using ::testing::FloatNear; - -constexpr float kMaxError = 1e-4; - -MATCHER_P2(PointEq, x, y, "") { - bool result = testing::Value(arg.x(), FloatNear(x, kMaxError)) && - testing::Value(arg.y(), FloatNear(y, kMaxError)); - if (!result) { - *result_listener << "actual: {" << arg.x() << ", " << arg.y() - << "}, expected: {" << x << ", " << y << "}"; - } - return result; -} - -MATCHER_P4(BoundingBoxEq, xmin, ymin, width, height, "") { - return testing::Value(arg.xmin(), FloatNear(xmin, kMaxError)) && - testing::Value(arg.ymin(), FloatNear(ymin, kMaxError)) && - testing::Value(arg.width(), FloatNear(width, kMaxError)) && - testing::Value(arg.height(), FloatNear(height, kMaxError)); -} - -std::vector GetPoints(const Detection& detection) { - std::vector points; - const auto& location_data = detection.location_data(); - for (int i = 0; i < location_data.relative_keypoints_size(); ++i) { - const auto& kp = location_data.relative_keypoints(i); - points.push_back({kp.x(), kp.y()}); - } - return points; -} - -// Test helper function to run "DetectionProjectionCalculator". -absl::StatusOr RunProjectionCalculator( - Detection detection, std::array project_mat) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionProjectionCalculator" - input_stream: "DETECTIONS:detections" - input_stream: "PROJECTION_MATRIX:matrix" - output_stream: "DETECTIONS:projected_detections" - )pb")); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back(MakePacket>( - std::vector({std::move(detection)})) - .At(Timestamp::PostStream())); - runner.MutableInputs() - ->Tag("PROJECTION_MATRIX") - .packets.push_back( - MakePacket>(std::move(project_mat)) - .At(Timestamp::PostStream())); - - MP_RETURN_IF_ERROR(runner.Run()); - const std::vector& output = - runner.Outputs().Tag("DETECTIONS").packets; - RET_CHECK_EQ(output.size(), 1); - const auto& output_detections = output[0].Get>(); - - RET_CHECK_EQ(output_detections.size(), 1); - return output_detections[0]; -} - -TEST(DetectionProjectionCalculatorTest, ProjectionFullRoiNoOp) { - Detection detection; - auto* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(0.0f); - location_data->mutable_relative_bounding_box()->set_ymin(0.0f); - location_data->mutable_relative_bounding_box()->set_width(0.5f); - location_data->mutable_relative_bounding_box()->set_height(0.5f); - - auto* kp = location_data->add_relative_keypoints(); - kp->set_x(0.25f); - kp->set_y(0.25f); - - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.0f); - roi.set_height(1.0f); - roi.set_rotation(0.0f); - - constexpr int kImageWidth = 100; - constexpr int kImageHeight = 100; - - RotatedRect rect; - rect.center_x = roi.x_center() * kImageWidth; - rect.center_y = roi.y_center() * kImageHeight; - rect.width = roi.width() * kImageWidth; - rect.height = roi.height() * kImageHeight; - rect.rotation = roi.rotation(); - - std::array projection_matrix; - GetRotatedSubRectToRectTransformMatrix(rect, kImageWidth, kImageHeight, - /*flip_horizontaly=*/false, - &projection_matrix); - - auto status_or_result = RunProjectionCalculator(std::move(detection), - std::move(projection_matrix)); - MP_ASSERT_OK(status_or_result); - const auto& result = status_or_result.value(); - ASSERT_EQ(result.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(result.location_data().relative_bounding_box(), - BoundingBoxEq(0.0f, 0.0f, 0.5f, 0.5f)); - EXPECT_THAT(GetPoints(result), testing::ElementsAre(PointEq(0.25f, 0.25f))); -} - -TEST(DetectionProjectionCalculatorTest, ProjectionFullRoi90Rotation) { - Detection detection; - auto* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(0.0f); - location_data->mutable_relative_bounding_box()->set_ymin(0.0f); - location_data->mutable_relative_bounding_box()->set_width(0.5f); - location_data->mutable_relative_bounding_box()->set_height(0.5f); - - auto* kp = location_data->add_relative_keypoints(); - kp->set_x(0.25f); - kp->set_y(0.25f); - - mediapipe::NormalizedRect roi; - roi.set_x_center(0.5f); - roi.set_y_center(0.5f); - roi.set_width(1.0f); - roi.set_height(1.0f); - roi.set_rotation(90 * M_PI / 180.0f); - - constexpr int kImageWidth = 100; - constexpr int kImageHeight = 100; - - RotatedRect rect; - rect.center_x = roi.x_center() * kImageWidth; - rect.center_y = roi.y_center() * kImageHeight; - rect.width = roi.width() * kImageWidth; - rect.height = roi.height() * kImageHeight; - rect.rotation = roi.rotation(); - - std::array projection_matrix; - GetRotatedSubRectToRectTransformMatrix(rect, kImageWidth, kImageHeight, - /*flip_horizontaly=*/false, - &projection_matrix); - - auto status_or_result = RunProjectionCalculator(std::move(detection), - std::move(projection_matrix)); - MP_ASSERT_OK(status_or_result); - const auto& result = status_or_result.value(); - ASSERT_EQ(result.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(result.location_data().relative_bounding_box(), - BoundingBoxEq(0.5f, 0.0f, 0.5f, 0.5f)); - EXPECT_THAT(GetPoints(result), ElementsAre(PointEq(0.75f, 0.25f))); -} - -TEST(DetectionProjectionCalculatorTest, ProjectionSmallerRoi) { - Detection detection; - auto* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(0.5f); - location_data->mutable_relative_bounding_box()->set_ymin(0.0f); - location_data->mutable_relative_bounding_box()->set_width(0.5f); - location_data->mutable_relative_bounding_box()->set_height(0.5f); - - auto* kp = location_data->add_relative_keypoints(); - kp->set_x(0.5f); - kp->set_y(0.5f); - - mediapipe::NormalizedRect roi; - roi.set_x_center(0.75f); - roi.set_y_center(0.75f); - roi.set_width(0.5f); - roi.set_height(0.5f); - roi.set_rotation(0.0f); - - constexpr int kImageWidth = 100; - constexpr int kImageHeight = 100; - - RotatedRect rect; - rect.center_x = roi.x_center() * kImageWidth; - rect.center_y = roi.y_center() * kImageHeight; - rect.width = roi.width() * kImageWidth; - rect.height = roi.height() * kImageHeight; - rect.rotation = roi.rotation(); - - std::array projection_matrix; - GetRotatedSubRectToRectTransformMatrix(rect, kImageWidth, kImageHeight, - /*flip_horizontaly=*/false, - &projection_matrix); - - auto status_or_result = RunProjectionCalculator(std::move(detection), - std::move(projection_matrix)); - MP_ASSERT_OK(status_or_result); - const auto& result = status_or_result.value(); - ASSERT_EQ(result.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(result.location_data().relative_bounding_box(), - BoundingBoxEq(0.75f, 0.5f, 0.25f, 0.25f)); - EXPECT_THAT(GetPoints(result), ElementsAre(PointEq(0.75f, 0.75f))); -} - -TEST(DetectionProjectionCalculatorTest, ProjectionSmallerRoi30Rotation) { - constexpr float kImageWidth = 80; - constexpr float kImageHeight = 120; - constexpr float kRectWidth = 50; - constexpr float kRectHeight = 30; - constexpr float kRectXCenter = 65; - constexpr float kRectYCenter = 85; - constexpr float kRectRotation = 30 * M_PI / 180.0f; - - Detection detection; - auto* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(0.0f); - location_data->mutable_relative_bounding_box()->set_ymin(0.0f); - location_data->mutable_relative_bounding_box()->set_width(1.0f); - location_data->mutable_relative_bounding_box()->set_height(1.0f); - // Expected box values were calculated manually from image. - constexpr float kExpectedBoxXMin = 35.849f / kImageWidth; - constexpr float kExpectedBoxYMin = 59.510f / kImageHeight; - constexpr float kExpectedBoxWidth = 58.301f / kImageWidth; - constexpr float kExpectedBoxHeight = 50.981f / kImageHeight; - - auto* kp1 = location_data->add_relative_keypoints(); - kp1->set_x(0.0f); - kp1->set_y(0.0f); - auto* kp2 = location_data->add_relative_keypoints(); - kp2->set_x(0.5f); - kp2->set_y(0.5f); - auto* kp3 = location_data->add_relative_keypoints(); - kp3->set_x(1.0f); - kp3->set_y(0.0f); - // Expected key points were calculated manually from image. - constexpr float kExpectedPoint1X = 50.85f / kImageWidth; - constexpr float kExpectedPoint1Y = 59.52f / kImageHeight; - constexpr float kExpectedPoint2X = kRectXCenter / kImageWidth; - constexpr float kExpectedPoint2Y = kRectYCenter / kImageHeight; - constexpr float kExpectedPoint3X = 94.15f / kImageWidth; - constexpr float kExpectedPoint3Y = 84.51f / kImageHeight; - - mediapipe::NormalizedRect roi; - roi.set_x_center(kRectXCenter / kImageWidth); - roi.set_y_center(kRectYCenter / kImageHeight); - roi.set_width(kRectWidth / kImageWidth); - roi.set_height(kRectHeight / kImageHeight); - roi.set_rotation(kRectRotation); - - RotatedRect rect; - rect.center_x = roi.x_center() * kImageWidth; - rect.center_y = roi.y_center() * kImageHeight; - rect.width = roi.width() * kImageWidth; - rect.height = roi.height() * kImageHeight; - rect.rotation = roi.rotation(); - - std::array projection_matrix; - GetRotatedSubRectToRectTransformMatrix(rect, kImageWidth, kImageHeight, - /*flip_horizontaly=*/false, - &projection_matrix); - - auto status_or_result = RunProjectionCalculator(std::move(detection), - std::move(projection_matrix)); - MP_ASSERT_OK(status_or_result); - const auto& result = status_or_result.value(); - ASSERT_EQ(result.location_data().format(), - LocationData::RELATIVE_BOUNDING_BOX); - EXPECT_THAT(result.location_data().relative_bounding_box(), - BoundingBoxEq(kExpectedBoxXMin, kExpectedBoxYMin, - kExpectedBoxWidth, kExpectedBoxHeight)); - EXPECT_THAT(GetPoints(result), - ElementsAre(PointEq(kExpectedPoint1X, kExpectedPoint1Y), - PointEq(kExpectedPoint2X, kExpectedPoint2Y), - PointEq(kExpectedPoint3X, kExpectedPoint3Y))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_to_landmarks_calculator.cc b/mediapipe/calculators/util/detection_to_landmarks_calculator.cc deleted file mode 100644 index 549298bad..000000000 --- a/mediapipe/calculators/util/detection_to_landmarks_calculator.cc +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionTag[] = "DETECTION"; -constexpr char kLandmarksTag[] = "LANDMARKS"; - -absl::Status ConvertDetectionToLandmarks(const Detection& detection, - NormalizedLandmarkList* landmarks) { - const auto& location_data = detection.location_data(); - for (int i = 0; i < location_data.relative_keypoints_size(); ++i) { - const auto& keypoint = location_data.relative_keypoints(i); - - auto* landmark = landmarks->add_landmark(); - landmark->set_x(keypoint.x()); - landmark->set_y(keypoint.y()); - } - - return absl::OkStatus(); -} - -} // namespace - -// Converts a detection into a normalized landmark list by extracting the -// location data relative keypoints as landmarks. -// -// Input: -// DETECTION - `Detection` -// A detection to be converted. -// -// Output: -// LANDMARKS - `NormalizedLandmarkList` -// A converted normalized landmark list. -// -// Example: -// -// node { -// calculator: "DetectionToLandmarksCalculator" -// input_stream: "DETECTION:detection" -// output_stream: "LANDMARKS:landmarks" -// } -// -class DetectionToLandmarksCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionTag)); - RET_CHECK(cc->Outputs().HasTag(kLandmarksTag)); - - cc->Inputs().Tag(kDetectionTag).Set(); - cc->Outputs().Tag(kLandmarksTag).Set(); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - const auto& detection = cc->Inputs().Tag(kDetectionTag).Get(); - - auto landmarks = absl::make_unique(); - MP_RETURN_IF_ERROR(ConvertDetectionToLandmarks(detection, landmarks.get())); - - cc->Outputs() - .Tag(kLandmarksTag) - .Add(landmarks.release(), cc->InputTimestamp()); - - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(DetectionToLandmarksCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detection_unique_id_calculator.cc b/mediapipe/calculators/util/detection_unique_id_calculator.cc deleted file mode 100644 index ac8889ffb..000000000 --- a/mediapipe/calculators/util/detection_unique_id_calculator.cc +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kDetectionListTag[] = "DETECTION_LIST"; - -// Each detection processed by DetectionUniqueIDCalculator will be assigned an -// unique id that starts from 1. If a detection already has an ID other than 0, -// the ID will be overwritten. -static int64 detection_id = 0; - -inline int GetNextDetectionId() { return ++detection_id; } - -} // namespace - -// Assign a unique id to detections. -// Note that the calculator will consume the input vector of Detection or -// DetectionList. So the input stream can not be connected to other calculators. -// -// Example config: -// node { -// calculator: "DetectionUniqueIdCalculator" -// input_stream: "DETECTIONS:detections" -// output_stream: "DETECTIONS:output_detections" -// } -class DetectionUniqueIdCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) || - cc->Inputs().HasTag(kDetectionsTag)) - << "None of the input streams are provided."; - - if (cc->Inputs().HasTag(kDetectionListTag)) { - RET_CHECK(cc->Outputs().HasTag(kDetectionListTag)); - cc->Inputs().Tag(kDetectionListTag).Set(); - cc->Outputs().Tag(kDetectionListTag).Set(); - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - RET_CHECK(cc->Outputs().HasTag(kDetectionsTag)); - cc->Inputs().Tag(kDetectionsTag).Set>(); - cc->Outputs().Tag(kDetectionsTag).Set>(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(mediapipe::TimestampDiff(0)); - return absl::OkStatus(); - } - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(DetectionUniqueIdCalculator); - -absl::Status DetectionUniqueIdCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kDetectionListTag) && - !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) { - auto result = - cc->Inputs().Tag(kDetectionListTag).Value().Consume(); - if (result.ok()) { - auto detection_list = std::move(result).value(); - for (Detection& detection : *detection_list->mutable_detection()) { - detection.set_detection_id(GetNextDetectionId()); - } - cc->Outputs() - .Tag(kDetectionListTag) - .Add(detection_list.release(), cc->InputTimestamp()); - } - } - - if (cc->Inputs().HasTag(kDetectionsTag) && - !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) { - auto result = cc->Inputs() - .Tag(kDetectionsTag) - .Value() - .Consume>(); - if (result.ok()) { - auto detections = std::move(result).value(); - for (Detection& detection : *detections) { - detection.set_detection_id(GetNextDetectionId()); - } - cc->Outputs() - .Tag(kDetectionsTag) - .Add(detections.release(), cc->InputTimestamp()); - } - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.cc b/mediapipe/calculators/util/detections_to_rects_calculator.cc deleted file mode 100644 index a1b13a8db..000000000 --- a/mediapipe/calculators/util/detections_to_rects_calculator.cc +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "mediapipe/calculators/util/detections_to_rects_calculator.h" - -#include -#include - -#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionTag[] = "DETECTION"; -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kImageSizeTag[] = "IMAGE_SIZE"; -constexpr char kRectTag[] = "RECT"; -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kRectsTag[] = "RECTS"; -constexpr char kNormRectsTag[] = "NORM_RECTS"; - -constexpr float kMinFloat = std::numeric_limits::lowest(); -constexpr float kMaxFloat = std::numeric_limits::max(); - -absl::Status NormRectFromKeyPoints(const LocationData& location_data, - NormalizedRect* rect) { - RET_CHECK_GT(location_data.relative_keypoints_size(), 1) - << "2 or more key points required to calculate a rect."; - float xmin = kMaxFloat; - float ymin = kMaxFloat; - float xmax = kMinFloat; - float ymax = kMinFloat; - for (int i = 0; i < location_data.relative_keypoints_size(); ++i) { - const auto& kp = location_data.relative_keypoints(i); - xmin = std::min(xmin, kp.x()); - ymin = std::min(ymin, kp.y()); - xmax = std::max(xmax, kp.x()); - ymax = std::max(ymax, kp.y()); - } - rect->set_x_center((xmin + xmax) / 2); - rect->set_y_center((ymin + ymax) / 2); - rect->set_width(xmax - xmin); - rect->set_height(ymax - ymin); - return absl::OkStatus(); -} - -template -void RectFromBox(B box, R* rect) { - rect->set_x_center(box.xmin() + box.width() / 2); - rect->set_y_center(box.ymin() + box.height() / 2); - rect->set_width(box.width()); - rect->set_height(box.height()); -} - -} // namespace - -absl::Status DetectionsToRectsCalculator::DetectionToRect( - const Detection& detection, const DetectionSpec& detection_spec, - Rect* rect) { - const LocationData location_data = detection.location_data(); - switch (options_.conversion_mode()) { - case mediapipe::DetectionsToRectsCalculatorOptions_ConversionMode_DEFAULT: - case mediapipe:: - DetectionsToRectsCalculatorOptions_ConversionMode_USE_BOUNDING_BOX: { - RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX) - << "Only Detection with formats of BOUNDING_BOX can be converted to " - "Rect"; - RectFromBox(location_data.bounding_box(), rect); - break; - } - case mediapipe:: - DetectionsToRectsCalculatorOptions_ConversionMode_USE_KEYPOINTS: { - RET_CHECK(detection_spec.image_size.has_value()) - << "Rect with absolute coordinates calculation requires image size."; - const int width = detection_spec.image_size->first; - const int height = detection_spec.image_size->second; - NormalizedRect norm_rect; - MP_RETURN_IF_ERROR(NormRectFromKeyPoints(location_data, &norm_rect)); - rect->set_x_center(std::round(norm_rect.x_center() * width)); - rect->set_y_center(std::round(norm_rect.y_center() * height)); - rect->set_width(std::round(norm_rect.width() * width)); - rect->set_height(std::round(norm_rect.height() * height)); - break; - } - } - return absl::OkStatus(); -} - -absl::Status DetectionsToRectsCalculator::DetectionToNormalizedRect( - const Detection& detection, const DetectionSpec& detection_spec, - NormalizedRect* rect) { - const LocationData location_data = detection.location_data(); - switch (options_.conversion_mode()) { - case mediapipe::DetectionsToRectsCalculatorOptions_ConversionMode_DEFAULT: - case mediapipe:: - DetectionsToRectsCalculatorOptions_ConversionMode_USE_BOUNDING_BOX: { - RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX) - << "Only Detection with formats of RELATIVE_BOUNDING_BOX can be " - "converted to NormalizedRect"; - RectFromBox(location_data.relative_bounding_box(), rect); - break; - } - case mediapipe:: - DetectionsToRectsCalculatorOptions_ConversionMode_USE_KEYPOINTS: { - MP_RETURN_IF_ERROR(NormRectFromKeyPoints(location_data, rect)); - break; - } - } - return absl::OkStatus(); -} - -absl::Status DetectionsToRectsCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionTag) ^ - cc->Inputs().HasTag(kDetectionsTag)) - << "Exactly one of DETECTION or DETECTIONS input stream should be " - "provided."; - RET_CHECK_EQ((cc->Outputs().HasTag(kNormRectTag) ? 1 : 0) + - (cc->Outputs().HasTag(kRectTag) ? 1 : 0) + - (cc->Outputs().HasTag(kNormRectsTag) ? 1 : 0) + - (cc->Outputs().HasTag(kRectsTag) ? 1 : 0), - 1) - << "Exactly one of NORM_RECT, RECT, NORM_RECTS or RECTS output stream " - "should be provided."; - - if (cc->Inputs().HasTag(kDetectionTag)) { - cc->Inputs().Tag(kDetectionTag).Set(); - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - cc->Inputs().Tag(kDetectionsTag).Set>(); - } - if (cc->Inputs().HasTag(kImageSizeTag)) { - cc->Inputs().Tag(kImageSizeTag).Set>(); - } - - if (cc->Outputs().HasTag(kRectTag)) { - cc->Outputs().Tag(kRectTag).Set(); - } - if (cc->Outputs().HasTag(kNormRectTag)) { - cc->Outputs().Tag(kNormRectTag).Set(); - } - if (cc->Outputs().HasTag(kRectsTag)) { - cc->Outputs().Tag(kRectsTag).Set>(); - } - if (cc->Outputs().HasTag(kNormRectsTag)) { - cc->Outputs().Tag(kNormRectsTag).Set>(); - } - - return absl::OkStatus(); -} - -absl::Status DetectionsToRectsCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - - if (options_.has_rotation_vector_start_keypoint_index()) { - RET_CHECK(options_.has_rotation_vector_end_keypoint_index()); - RET_CHECK(options_.has_rotation_vector_target_angle() ^ - options_.has_rotation_vector_target_angle_degrees()); - RET_CHECK(cc->Inputs().HasTag(kImageSizeTag)); - - if (options_.has_rotation_vector_target_angle()) { - target_angle_ = options_.rotation_vector_target_angle(); - } else { - target_angle_ = - M_PI * options_.rotation_vector_target_angle_degrees() / 180.f; - } - start_keypoint_index_ = options_.rotation_vector_start_keypoint_index(); - end_keypoint_index_ = options_.rotation_vector_end_keypoint_index(); - rotate_ = true; - } - - output_zero_rect_for_empty_detections_ = - options_.output_zero_rect_for_empty_detections(); - - return absl::OkStatus(); -} - -absl::Status DetectionsToRectsCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kDetectionTag) && - cc->Inputs().Tag(kDetectionTag).IsEmpty()) { - return absl::OkStatus(); - } - if (cc->Inputs().HasTag(kDetectionsTag) && - cc->Inputs().Tag(kDetectionsTag).IsEmpty()) { - return absl::OkStatus(); - } - - std::vector detections; - if (cc->Inputs().HasTag(kDetectionTag)) { - detections.push_back(cc->Inputs().Tag(kDetectionTag).Get()); - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - detections = cc->Inputs().Tag(kDetectionsTag).Get>(); - if (detections.empty()) { - if (output_zero_rect_for_empty_detections_) { - if (cc->Outputs().HasTag(kRectTag)) { - cc->Outputs().Tag(kRectTag).AddPacket( - MakePacket().At(cc->InputTimestamp())); - } - if (cc->Outputs().HasTag(kNormRectTag)) { - cc->Outputs() - .Tag(kNormRectTag) - .AddPacket(MakePacket().At(cc->InputTimestamp())); - } - if (cc->Outputs().HasTag(kNormRectsTag)) { - auto rect_vector = absl::make_unique>(); - rect_vector->emplace_back(NormalizedRect()); - cc->Outputs() - .Tag(kNormRectsTag) - .Add(rect_vector.release(), cc->InputTimestamp()); - } - } - return absl::OkStatus(); - } - } - - // Get dynamic calculator options (e.g. `image_size`). - const DetectionSpec detection_spec = GetDetectionSpec(cc); - - if (cc->Outputs().HasTag(kRectTag)) { - auto output_rect = absl::make_unique(); - MP_RETURN_IF_ERROR( - DetectionToRect(detections[0], detection_spec, output_rect.get())); - if (rotate_) { - float rotation; - MP_RETURN_IF_ERROR( - ComputeRotation(detections[0], detection_spec, &rotation)); - output_rect->set_rotation(rotation); - } - cc->Outputs().Tag(kRectTag).Add(output_rect.release(), - cc->InputTimestamp()); - } - if (cc->Outputs().HasTag(kNormRectTag)) { - auto output_rect = absl::make_unique(); - MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec, - output_rect.get())); - if (rotate_) { - float rotation; - MP_RETURN_IF_ERROR( - ComputeRotation(detections[0], detection_spec, &rotation)); - output_rect->set_rotation(rotation); - } - cc->Outputs() - .Tag(kNormRectTag) - .Add(output_rect.release(), cc->InputTimestamp()); - } - if (cc->Outputs().HasTag(kRectsTag)) { - auto output_rects = absl::make_unique>(detections.size()); - for (int i = 0; i < detections.size(); ++i) { - MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec, - &(output_rects->at(i)))); - if (rotate_) { - float rotation; - MP_RETURN_IF_ERROR( - ComputeRotation(detections[i], detection_spec, &rotation)); - output_rects->at(i).set_rotation(rotation); - } - } - cc->Outputs().Tag(kRectsTag).Add(output_rects.release(), - cc->InputTimestamp()); - } - if (cc->Outputs().HasTag(kNormRectsTag)) { - auto output_rects = - absl::make_unique>(detections.size()); - for (int i = 0; i < detections.size(); ++i) { - MP_RETURN_IF_ERROR(DetectionToNormalizedRect( - detections[i], detection_spec, &(output_rects->at(i)))); - if (rotate_) { - float rotation; - MP_RETURN_IF_ERROR( - ComputeRotation(detections[i], detection_spec, &rotation)); - output_rects->at(i).set_rotation(rotation); - } - } - cc->Outputs() - .Tag(kNormRectsTag) - .Add(output_rects.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status DetectionsToRectsCalculator::ComputeRotation( - const Detection& detection, const DetectionSpec& detection_spec, - float* rotation) { - const auto& location_data = detection.location_data(); - const auto& image_size = detection_spec.image_size; - RET_CHECK(image_size) << "Image size is required to calculate rotation"; - - const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() * - image_size->first; - const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() * - image_size->second; - const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() * - image_size->first; - const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() * - image_size->second; - - *rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0)); - - return absl::OkStatus(); -} - -DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec( - const CalculatorContext* cc) { - absl::optional> image_size; - if (HasTagValue(cc->Inputs(), kImageSizeTag)) { - image_size = cc->Inputs().Tag(kImageSizeTag).Get>(); - } - - return {image_size}; -} - -REGISTER_CALCULATOR(DetectionsToRectsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.h b/mediapipe/calculators/util/detections_to_rects_calculator.h deleted file mode 100644 index e91441bc6..000000000 --- a/mediapipe/calculators/util/detections_to_rects_calculator.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef MEDIAPIPE_CALCULATORS_UTIL_DETECTIONS_TO_RECTS_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_DETECTIONS_TO_RECTS_CALCULATOR_H_ - -#include - -#include "absl/types/optional.h" -#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// Dynamic options passed as calculator `input_stream` that can be used for -// calculation of rectangle or rotation for given detection. Does not include -// static calculator options which are available via private fields. -struct DetectionSpec { - absl::optional> image_size; -}; - -// A calculator that converts Detection proto to Rect proto. -// -// Detection is the format for encoding one or more detections in an image. -// The input can be a single Detection or std::vector. The output can -// be either a single Rect or NormalizedRect, or std::vector or -// std::vector. If Rect is used, the LocationData format is -// expected to be BOUNDING_BOX, and if NormalizedRect is used it is expected to -// be RELATIVE_BOUNDING_BOX. -// -// When the input is std::vector and the output is a Rect or -// NormalizedRect, only the first detection is converted. When the input is a -// single Detection and the output is a std::vector or -// std::vector, the output is a vector of size 1. -// -// Inputs: -// -// One of the following: -// DETECTION: A Detection proto. -// DETECTIONS: An std::vector. -// -// IMAGE_SIZE (optional): A std::pair represention image width and -// height. This is required only when rotation needs to be computed (see -// calculator options). -// -// Output: -// One of the following: -// RECT: A Rect proto. -// NORM_RECT: A NormalizedRect proto. -// RECTS: An std::vector. -// NORM_RECTS: An std::vector. -// -// Example config: -// node { -// calculator: "DetectionsToRectsCalculator" -// input_stream: "DETECTIONS:detections" -// input_stream: "IMAGE_SIZE:image_size" -// output_stream: "NORM_RECT:rect" -// options: { -// [mediapipe.DetectionsToRectCalculatorOptions.ext] { -// rotation_vector_start_keypoint_index: 0 -// rotation_vector_end_keypoint_index: 2 -// rotation_vector_target_angle_degrees: 90 -// output_zero_rect_for_empty_detections: true -// } -// } -// } -class DetectionsToRectsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - protected: - virtual absl::Status DetectionToRect(const ::mediapipe::Detection& detection, - const DetectionSpec& detection_spec, - ::mediapipe::Rect* rect); - virtual absl::Status DetectionToNormalizedRect( - const ::mediapipe::Detection& detection, - const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect); - virtual absl::Status ComputeRotation(const ::mediapipe::Detection& detection, - const DetectionSpec& detection_spec, - float* rotation); - virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc); - - static inline float NormalizeRadians(float angle) { - return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI)); - } - - ::mediapipe::DetectionsToRectsCalculatorOptions options_; - int start_keypoint_index_; - int end_keypoint_index_; - float target_angle_ = 0.0f; // In radians. - bool rotate_; - bool output_zero_rect_for_empty_detections_; -}; - -} // namespace mediapipe -#endif // MEDIAPIPE_CALCULATORS_UTIL_DETECTIONS_TO_RECTS_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.proto b/mediapipe/calculators/util/detections_to_rects_calculator.proto deleted file mode 100644 index d49eb6c52..000000000 --- a/mediapipe/calculators/util/detections_to_rects_calculator.proto +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message DetectionsToRectsCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional DetectionsToRectsCalculatorOptions ext = 262691807; - } - - // Specify the rotation angle of the output rect with a vector formed by - // connecting two keypoints in the detection, together with the target angle - // (can be in radians or in degrees) of that vector after rotation. The target - // angle is counter-clockwise starting from the positive x-axis. - optional int32 rotation_vector_start_keypoint_index = 1; - optional int32 rotation_vector_end_keypoint_index = 2; - optional float rotation_vector_target_angle = 3; // In radians. - optional float rotation_vector_target_angle_degrees = 4; // In degrees. - - // Whether to output a zero-rect (with origin and size both zero) when the - // input detection vector is empty. - optional bool output_zero_rect_for_empty_detections = 5; - - enum ConversionMode { - DEFAULT = 0; - USE_BOUNDING_BOX = 1; - USE_KEYPOINTS = 2; - } - - optional ConversionMode conversion_mode = 6; -} diff --git a/mediapipe/calculators/util/detections_to_rects_calculator_test.cc b/mediapipe/calculators/util/detections_to_rects_calculator_test.cc deleted file mode 100644 index 3eae1af9d..000000000 --- a/mediapipe/calculators/util/detections_to_rects_calculator_test.cc +++ /dev/null @@ -1,424 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -MATCHER_P4(RectEq, x_center, y_center, width, height, "") { - return testing::Value(arg.x_center(), testing::Eq(x_center)) && - testing::Value(arg.y_center(), testing::Eq(y_center)) && - testing::Value(arg.width(), testing::Eq(width)) && - testing::Value(arg.height(), testing::Eq(height)); -} - -MATCHER_P4(NormRectEq, x_center, y_center, width, height, "") { - return testing::Value(arg.x_center(), testing::FloatEq(x_center)) && - testing::Value(arg.y_center(), testing::FloatEq(y_center)) && - testing::Value(arg.width(), testing::FloatEq(width)) && - testing::Value(arg.height(), testing::FloatEq(height)); -} - -Detection DetectionWithLocationData(int32 xmin, int32 ymin, int32 width, - int32 height) { - Detection detection; - LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::BOUNDING_BOX); - location_data->mutable_bounding_box()->set_xmin(xmin); - location_data->mutable_bounding_box()->set_ymin(ymin); - location_data->mutable_bounding_box()->set_width(width); - location_data->mutable_bounding_box()->set_height(height); - return detection; -} - -Detection DetectionWithKeyPoints( - const std::vector>& key_points) { - Detection detection; - LocationData* location_data = detection.mutable_location_data(); - std::for_each(key_points.begin(), key_points.end(), - [location_data](std::pair kp) { - auto* new_kp = location_data->add_relative_keypoints(); - new_kp->set_x(kp.first); - new_kp->set_y(kp.second); - }); - return detection; -} - -Detection DetectionWithRelativeLocationData(double xmin, double ymin, - double width, double height) { - Detection detection; - LocationData* location_data = detection.mutable_location_data(); - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data->mutable_relative_bounding_box()->set_xmin(xmin); - location_data->mutable_relative_bounding_box()->set_ymin(ymin); - location_data->mutable_relative_bounding_box()->set_width(width); - location_data->mutable_relative_bounding_box()->set_height(height); - return detection; -} - -TEST(DetectionsToRectsCalculatorTest, DetectionToRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - output_stream: "RECT:rect" - )pb")); - - auto detection = absl::make_unique( - DetectionWithLocationData(100, 200, 300, 400)); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back( - Adopt(detection.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("RECT").packets; - ASSERT_EQ(1, output.size()); - const auto& rect = output[0].Get(); - EXPECT_THAT(rect, RectEq(250, 400, 300, 400)); -} - -absl::StatusOr RunDetectionKeyPointsToRectCalculation( - Detection detection, std::pair image_size) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - input_stream: "IMAGE_SIZE:image_size" - output_stream: "RECT:rect" - options: { - [mediapipe.DetectionsToRectsCalculatorOptions.ext] { - conversion_mode: USE_KEYPOINTS - } - } - )pb")); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back(MakePacket(std::move(detection)) - .At(Timestamp::PostStream())); - runner.MutableInputs() - ->Tag("IMAGE_SIZE") - .packets.push_back(MakePacket>(image_size) - .At(Timestamp::PostStream())); - - MP_RETURN_IF_ERROR(runner.Run()); - const std::vector& output = runner.Outputs().Tag("RECT").packets; - RET_CHECK_EQ(output.size(), 1); - return output[0].Get(); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToRect) { - auto status_or_value = RunDetectionKeyPointsToRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.0f, 0.0f}, {1.0f, 1.0f}}), - /*image_size=*/{640, 480}); - EXPECT_THAT(status_or_value.value(), RectEq(320, 240, 640, 480)); - - status_or_value = RunDetectionKeyPointsToRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.25f, 0.25f}, {0.75f, 0.75f}}), - /*image_size=*/{640, 480}); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(320, 240, 320, 240)); - - status_or_value = RunDetectionKeyPointsToRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.0f, 0.0f}, {0.5f, 0.5f}}), - /*image_size=*/{640, 480}); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(160, 120, 320, 240)); - - status_or_value = RunDetectionKeyPointsToRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.5f, 0.5f}, {1.0f, 1.0f}}), - /*image_size=*/{640, 480}); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(480, 360, 320, 240)); - - status_or_value = RunDetectionKeyPointsToRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.25f, 0.25f}, {0.75f, 0.75f}}), - /*image_size=*/{0, 0}); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(0, 0, 0, 0)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - output_stream: "NORM_RECT:rect" - )pb")); - - auto detection = absl::make_unique( - DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4)); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back( - Adopt(detection.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("NORM_RECT").packets; - ASSERT_EQ(1, output.size()); - const auto& rect = output[0].Get(); - EXPECT_THAT(rect, NormRectEq(0.25f, 0.4f, 0.3f, 0.4f)); -} - -absl::StatusOr RunDetectionKeyPointsToNormRectCalculation( - Detection detection) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - output_stream: "NORM_RECT:rect" - options: { - [mediapipe.DetectionsToRectsCalculatorOptions.ext] { - conversion_mode: USE_KEYPOINTS - } - } - )pb")); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back(MakePacket(std::move(detection)) - .At(Timestamp::PostStream())); - - MP_RETURN_IF_ERROR(runner.Run()); - const std::vector& output = runner.Outputs().Tag("NORM_RECT").packets; - RET_CHECK_EQ(output.size(), 1); - return output[0].Get(); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionKeyPointsToNormalizedRect) { - NormalizedRect rect; - - auto status_or_value = RunDetectionKeyPointsToNormRectCalculation( - /*detection=*/DetectionWithKeyPoints( - {{0.0f, 0.0f}, {0.5f, 0.5f}, {1.0f, 1.0f}})); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(0.5f, 0.5f, 1.0f, 1.0f)); - - status_or_value = RunDetectionKeyPointsToNormRectCalculation( - /*detection=*/DetectionWithKeyPoints( - {{0.25f, 0.25f}, {0.75f, 0.25f}, {0.75f, 0.75f}})); - EXPECT_THAT(status_or_value.value(), RectEq(0.5f, 0.5f, 0.5f, 0.5f)); - - status_or_value = RunDetectionKeyPointsToNormRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.0f, 0.0f}, {0.5f, 0.5f}})); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(0.25f, 0.25f, 0.5f, 0.5f)); - - status_or_value = RunDetectionKeyPointsToNormRectCalculation( - /*detection=*/DetectionWithKeyPoints({{0.5f, 0.5f}, {1.0f, 1.0f}})); - MP_ASSERT_OK(status_or_value); - EXPECT_THAT(status_or_value.value(), RectEq(0.75f, 0.75f, 0.5f, 0.5f)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionsToRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "RECT:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithLocationData(100, 200, 300, 400)); - detections->push_back(DetectionWithLocationData(200, 300, 400, 500)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("RECT").packets; - ASSERT_EQ(1, output.size()); - const auto& rect = output[0].Get(); - EXPECT_THAT(rect, RectEq(250, 400, 300, 400)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "NORM_RECT:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4)); - detections->push_back(DetectionWithRelativeLocationData(0.2, 0.3, 0.4, 0.5)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("NORM_RECT").packets; - ASSERT_EQ(1, output.size()); - const auto& rect = output[0].Get(); - EXPECT_THAT(rect, NormRectEq(0.25f, 0.4f, 0.3f, 0.4f)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionsToRects) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "RECTS:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithLocationData(100, 200, 300, 400)); - detections->push_back(DetectionWithLocationData(200, 300, 400, 500)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("RECTS").packets; - ASSERT_EQ(1, output.size()); - const auto& rects = output[0].Get>(); - ASSERT_EQ(rects.size(), 2); - EXPECT_THAT(rects[0], RectEq(250, 400, 300, 400)); - EXPECT_THAT(rects[1], RectEq(400, 550, 400, 500)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionsToNormalizedRects) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "NORM_RECTS:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4)); - detections->push_back(DetectionWithRelativeLocationData(0.2, 0.3, 0.4, 0.5)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("NORM_RECTS").packets; - ASSERT_EQ(1, output.size()); - const auto& rects = output[0].Get>(); - ASSERT_EQ(rects.size(), 2); - EXPECT_THAT(rects[0], NormRectEq(0.25f, 0.4f, 0.3f, 0.4f)); - EXPECT_THAT(rects[1], NormRectEq(0.4f, 0.55f, 0.4f, 0.5f)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionToRects) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - output_stream: "RECTS:rect" - )pb")); - - auto detection = absl::make_unique( - DetectionWithLocationData(100, 200, 300, 400)); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back( - Adopt(detection.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("RECTS").packets; - ASSERT_EQ(1, output.size()); - const auto& rects = output[0].Get>(); - EXPECT_EQ(rects.size(), 1); - EXPECT_THAT(rects[0], RectEq(250, 400, 300, 400)); -} - -TEST(DetectionsToRectsCalculatorTest, DetectionToNormalizedRects) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTION:detection" - output_stream: "NORM_RECTS:rect" - )pb")); - - auto detection = absl::make_unique( - DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4)); - - runner.MutableInputs() - ->Tag("DETECTION") - .packets.push_back( - Adopt(detection.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("NORM_RECTS").packets; - ASSERT_EQ(1, output.size()); - const auto& rects = output[0].Get>(); - ASSERT_EQ(rects.size(), 1); - EXPECT_THAT(rects[0], NormRectEq(0.25f, 0.4f, 0.3f, 0.4f)); -} - -TEST(DetectionsToRectsCalculatorTest, WrongInputToRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "RECT:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithRelativeLocationData(0.1, 0.2, 0.3, 0.4)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - ASSERT_THAT( - runner.Run().message(), - testing::HasSubstr("Only Detection with formats of BOUNDING_BOX")); -} - -TEST(DetectionsToRectsCalculatorTest, WrongInputToNormalizedRect) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRectsCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "NORM_RECT:rect" - )pb")); - - auto detections(absl::make_unique>()); - detections->push_back(DetectionWithLocationData(100, 200, 300, 400)); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - ASSERT_THAT(runner.Run().message(), - testing::HasSubstr( - "Only Detection with formats of RELATIVE_BOUNDING_BOX")); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detections_to_render_data_calculator.cc b/mediapipe/calculators/util/detections_to_render_data_calculator.cc deleted file mode 100644 index 25d74ba68..000000000 --- a/mediapipe/calculators/util/detections_to_render_data_calculator.cc +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "mediapipe/calculators/util/detections_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" -namespace mediapipe { - -namespace { - -constexpr char kDetectionTag[] = "DETECTION"; -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kDetectionListTag[] = "DETECTION_LIST"; -constexpr char kRenderDataTag[] = "RENDER_DATA"; - -constexpr char kSceneLabelLabel[] = "LABEL"; -constexpr char kSceneFeatureLabel[] = "FEATURE"; -constexpr char kSceneLocationLabel[] = "LOCATION"; -constexpr char kKeypointLabel[] = "KEYPOINT"; - -// The ratio of detection label font height to the height of detection bounding -// box. -constexpr double kLabelToBoundingBoxRatio = 0.1; -// Perserve 2 decimal digits. -constexpr float kNumScoreDecimalDigitsMultipler = 100; - -} // namespace - -// A calculator that converts Detection proto to RenderData proto for -// visualization. -// -// Detection is the format for encoding one or more detections in an image. -// The input can be std::vector or DetectionList. -// -// Please note that only Location Data formats of BOUNDING_BOX and -// RELATIVE_BOUNDING_BOX are supported. Normalized coordinates for -// RELATIVE_BOUNDING_BOX must be between 0.0 and 1.0. Any incremental normalized -// coordinates calculation in this calculator is capped at 1.0. -// -// The text(s) for "label(_id),score" will be shown on top left -// corner of the bounding box. The text for "feature_tag" will be shown on -// bottom left corner of the bounding box. -// -// Example config: -// node { -// calculator: "DetectionsToRenderDataCalculator" -// input_stream: "DETECTION:detection" -// input_stream: "DETECTIONS:detections" -// input_stream: "DETECTION_LIST:detection_list" -// output_stream: "RENDER_DATA:render_data" -// options { -// [DetectionsToRenderDataCalculatorOptions.ext] { -// produce_empty_packet : false -// } -// } -// } -class DetectionsToRenderDataCalculator : public CalculatorBase { - public: - DetectionsToRenderDataCalculator() {} - ~DetectionsToRenderDataCalculator() override {} - DetectionsToRenderDataCalculator(const DetectionsToRenderDataCalculator&) = - delete; - DetectionsToRenderDataCalculator& operator=( - const DetectionsToRenderDataCalculator&) = delete; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - // These utility methods are supposed to be used only by this class. No - // external client should depend on them. Due to C++ style guide unnamed - // namespace should not be used in header files. So, these has been defined - // as private static methods. - static void SetRenderAnnotationColorThickness( - const DetectionsToRenderDataCalculatorOptions& options, - RenderAnnotation* render_annotation); - - static void SetTextCoordinate(bool normalized, double left, double baseline, - RenderAnnotation::Text* text); - - static void SetRectCoordinate(bool normalized, double xmin, double ymin, - double width, double height, - RenderAnnotation::Rectangle* rect); - - static void AddLabels(const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - float text_line_height, RenderData* render_data); - static void AddFeatureTag( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - float text_line_height, RenderData* render_data); - static void AddLocationData( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - RenderData* render_data); - static void AddDetectionToRenderData( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - RenderData* render_data); -}; -REGISTER_CALCULATOR(DetectionsToRenderDataCalculator); - -absl::Status DetectionsToRenderDataCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) || - cc->Inputs().HasTag(kDetectionsTag) || - cc->Inputs().HasTag(kDetectionTag)) - << "None of the input streams are provided."; - - if (cc->Inputs().HasTag(kDetectionTag)) { - cc->Inputs().Tag(kDetectionTag).Set(); - } - if (cc->Inputs().HasTag(kDetectionListTag)) { - cc->Inputs().Tag(kDetectionListTag).Set(); - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - cc->Inputs().Tag(kDetectionsTag).Set>(); - } - cc->Outputs().Tag(kRenderDataTag).Set(); - return absl::OkStatus(); -} - -absl::Status DetectionsToRenderDataCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); -} - -absl::Status DetectionsToRenderDataCalculator::Process(CalculatorContext* cc) { - const auto& options = cc->Options(); - const bool has_detection_from_list = - cc->Inputs().HasTag(kDetectionListTag) && !cc->Inputs() - .Tag(kDetectionListTag) - .Get() - .detection() - .empty(); - const bool has_detection_from_vector = - cc->Inputs().HasTag(kDetectionsTag) && - !cc->Inputs().Tag(kDetectionsTag).Get>().empty(); - const bool has_single_detection = cc->Inputs().HasTag(kDetectionTag) && - !cc->Inputs().Tag(kDetectionTag).IsEmpty(); - if (!options.produce_empty_packet() && !has_detection_from_list && - !has_detection_from_vector && !has_single_detection) { - return absl::OkStatus(); - } - - // TODO: Add score threshold to - // DetectionsToRenderDataCalculatorOptions. - auto render_data = absl::make_unique(); - render_data->set_scene_class(options.scene_class()); - if (has_detection_from_list) { - for (const auto& detection : - cc->Inputs().Tag(kDetectionListTag).Get().detection()) { - AddDetectionToRenderData(detection, options, render_data.get()); - } - } - if (has_detection_from_vector) { - for (const auto& detection : - cc->Inputs().Tag(kDetectionsTag).Get>()) { - AddDetectionToRenderData(detection, options, render_data.get()); - } - } - if (has_single_detection) { - AddDetectionToRenderData(cc->Inputs().Tag(kDetectionTag).Get(), - options, render_data.get()); - } - cc->Outputs() - .Tag(kRenderDataTag) - .Add(render_data.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -void DetectionsToRenderDataCalculator::SetRenderAnnotationColorThickness( - const DetectionsToRenderDataCalculatorOptions& options, - RenderAnnotation* render_annotation) { - render_annotation->mutable_color()->set_r(options.color().r()); - render_annotation->mutable_color()->set_g(options.color().g()); - render_annotation->mutable_color()->set_b(options.color().b()); - render_annotation->set_thickness(options.thickness()); -} - -void DetectionsToRenderDataCalculator::SetTextCoordinate( - bool normalized, double left, double baseline, - RenderAnnotation::Text* text) { - text->set_normalized(normalized); - text->set_left(normalized ? std::max(left, 0.0) : left); - // Normalized coordinates must be between 0.0 and 1.0, if they are used. - text->set_baseline(normalized ? std::min(baseline, 1.0) : baseline); -} - -void DetectionsToRenderDataCalculator::SetRectCoordinate( - bool normalized, double xmin, double ymin, double width, double height, - RenderAnnotation::Rectangle* rect) { - if (xmin + width < 0.0 || ymin + height < 0.0) return; - if (normalized) { - if (xmin > 1.0 || ymin > 1.0) return; - } - rect->set_normalized(normalized); - rect->set_left(normalized ? std::max(xmin, 0.0) : xmin); - rect->set_top(normalized ? std::max(ymin, 0.0) : ymin); - // No "xmin + width -1" because the coordinates can be relative, i.e. [0,1], - // and we don't know what 1 pixel means in term of double [0,1]. - // For consistency decided to not decrease by 1 also when it is not relative. - // However, when the coordinate is normalized it has to be between 0.0 and - // 1.0. - rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width); - rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height); -} - -void DetectionsToRenderDataCalculator::AddLabels( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - float text_line_height, RenderData* render_data) { - CHECK(detection.label().empty() || detection.label_id().empty() || - detection.label_size() == detection.label_id_size()) - << "String or integer labels should be of same size. Or only one of them " - "is present."; - const auto num_labels = - std::max(detection.label_size(), detection.label_id_size()); - CHECK_EQ(detection.score_size(), num_labels) - << "Number of scores and labels should match for detection."; - - // Extracts all "label(_id),score" for the detection. - std::vector label_and_scores = {}; - for (int i = 0; i < num_labels; ++i) { - std::string label_str = detection.label().empty() - ? absl::StrCat(detection.label_id(i)) - : detection.label(i); - const float rounded_score = - std::round(detection.score(i) * kNumScoreDecimalDigitsMultipler) / - kNumScoreDecimalDigitsMultipler; - std::string label_and_score = - absl::StrCat(label_str, options.text_delimiter(), rounded_score, - options.text_delimiter()); - label_and_scores.push_back(label_and_score); - } - std::vector labels; - if (options.render_detection_id()) { - const std::string detection_id_str = - absl::StrCat("Id: ", detection.detection_id()); - labels.push_back(detection_id_str); - } - if (options.one_label_per_line()) { - labels.insert(labels.end(), label_and_scores.begin(), - label_and_scores.end()); - } else { - labels.push_back(absl::StrJoin(label_and_scores, "")); - } - // Add the render annotations for "label(_id),score". - for (int i = 0; i < labels.size(); ++i) { - auto label = labels.at(i); - auto* label_annotation = render_data->add_render_annotations(); - label_annotation->set_scene_tag(kSceneLabelLabel); - SetRenderAnnotationColorThickness(options, label_annotation); - auto* text = label_annotation->mutable_text(); - *text = options.text(); - text->set_display_text(label); - if (detection.location_data().format() == LocationData::BOUNDING_BOX) { - SetTextCoordinate(false, detection.location_data().bounding_box().xmin(), - detection.location_data().bounding_box().ymin() + - (i + 1) * text_line_height, - text); - } else { - text->set_font_height(text_line_height * 0.9); - SetTextCoordinate( - true, detection.location_data().relative_bounding_box().xmin(), - detection.location_data().relative_bounding_box().ymin() + - (i + 1) * text_line_height, - text); - } - } -} - -void DetectionsToRenderDataCalculator::AddFeatureTag( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - float text_line_height, RenderData* render_data) { - auto* feature_tag_annotation = render_data->add_render_annotations(); - feature_tag_annotation->set_scene_tag(kSceneFeatureLabel); - SetRenderAnnotationColorThickness(options, feature_tag_annotation); - auto* feature_tag_text = feature_tag_annotation->mutable_text(); - feature_tag_text->set_display_text(detection.feature_tag()); - if (detection.location_data().format() == LocationData::BOUNDING_BOX) { - SetTextCoordinate(false, detection.location_data().bounding_box().xmin(), - detection.location_data().bounding_box().ymin() + - detection.location_data().bounding_box().height(), - feature_tag_text); - } else { - feature_tag_text->set_font_height(text_line_height * 0.9); - SetTextCoordinate( - true, detection.location_data().relative_bounding_box().xmin(), - detection.location_data().relative_bounding_box().ymin() + - detection.location_data().relative_bounding_box().height(), - feature_tag_text); - } -} - -void DetectionsToRenderDataCalculator::AddLocationData( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - RenderData* render_data) { - auto* location_data_annotation = render_data->add_render_annotations(); - location_data_annotation->set_scene_tag(kSceneLocationLabel); - SetRenderAnnotationColorThickness(options, location_data_annotation); - auto* location_data_rect = location_data_annotation->mutable_rectangle(); - if (detection.location_data().format() == LocationData::BOUNDING_BOX) { - SetRectCoordinate(false, detection.location_data().bounding_box().xmin(), - detection.location_data().bounding_box().ymin(), - detection.location_data().bounding_box().width(), - detection.location_data().bounding_box().height(), - location_data_rect); - } else { - SetRectCoordinate( - true, detection.location_data().relative_bounding_box().xmin(), - detection.location_data().relative_bounding_box().ymin(), - detection.location_data().relative_bounding_box().width(), - detection.location_data().relative_bounding_box().height(), - location_data_rect); - // Keypoints are only supported in normalized/relative coordinates. - if (detection.location_data().relative_keypoints_size()) { - for (int i = 0; i < detection.location_data().relative_keypoints_size(); - ++i) { - auto* keypoint_data_annotation = render_data->add_render_annotations(); - keypoint_data_annotation->set_scene_tag(kKeypointLabel); - SetRenderAnnotationColorThickness(options, keypoint_data_annotation); - auto* keypoint_data = keypoint_data_annotation->mutable_point(); - keypoint_data->set_normalized(true); - // See location_data.proto for detail. - keypoint_data->set_x( - detection.location_data().relative_keypoints(i).x()); - keypoint_data->set_y( - detection.location_data().relative_keypoints(i).y()); - } - } - } -} - -void DetectionsToRenderDataCalculator::AddDetectionToRenderData( - const Detection& detection, - const DetectionsToRenderDataCalculatorOptions& options, - RenderData* render_data) { - CHECK(detection.location_data().format() == LocationData::BOUNDING_BOX || - detection.location_data().format() == - LocationData::RELATIVE_BOUNDING_BOX) - << "Only Detection with formats of BOUNDING_BOX or RELATIVE_BOUNDING_BOX " - "are supported."; - double text_line_height; - if (detection.location_data().format() == LocationData::BOUNDING_BOX) { - text_line_height = options.text().font_height(); - } else { - // Determine the text line height based on the default label to bounding box - // ratio and the number of labels. - text_line_height = - detection.location_data().relative_bounding_box().height() * - std::min(kLabelToBoundingBoxRatio, - 1 / (double)(std::max(detection.label_size(), - detection.label_id_size()) + - 1 /* for feature_tag */)); - } - AddLabels(detection, options, text_line_height, render_data); - AddFeatureTag(detection, options, text_line_height, render_data); - AddLocationData(detection, options, render_data); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detections_to_render_data_calculator.proto b/mediapipe/calculators/util/detections_to_render_data_calculator.proto deleted file mode 100644 index acc847dcd..000000000 --- a/mediapipe/calculators/util/detections_to_render_data_calculator.proto +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; -import "mediapipe/util/render_data.proto"; - -message DetectionsToRenderDataCalculatorOptions { - extend CalculatorOptions { - optional DetectionsToRenderDataCalculatorOptions ext = 248360806; - } - - // If true, produces a RenderData packet with no annotation when the input - // packet has no detection. Otherwise, it won't produce any packet. - // Please note, regardless of this flag nothing will be produce if there is - // no input packet for a timestamp. - optional bool produce_empty_packet = 1 [default = true]; - - // The delimiter to separate label(_id) and score. - optional string text_delimiter = 2 [default = ","]; - - // If true, each "label(_id),score" will be on a separate line. - // Otherwise, all "label(_id),score" will be concatenated when the detection - // has more than one label. - optional bool one_label_per_line = 3 [default = false]; - - // Rendering options for the label. - optional RenderAnnotation.Text text = 4; - - // Thickness for drawing the label(s) and the location_data(box). - optional double thickness = 5 [default = 1.0]; - - // Color for drawing the label(s), feature_tag, and the location_data(box). - optional Color color = 6; - - // An optional string that identifies this class of annotations - // for the render data output this calculator produces. If multiple - // instances of this calculator are present in the graph, this value - // should be unique among them. - optional string scene_class = 7 [default = "DETECTION"]; - - // If true, renders the detection id in the first line before the labels. - optional bool render_detection_id = 8 [default = false]; -} diff --git a/mediapipe/calculators/util/detections_to_render_data_calculator_test.cc b/mediapipe/calculators/util/detections_to_render_data_calculator_test.cc deleted file mode 100644 index ea4bfc484..000000000 --- a/mediapipe/calculators/util/detections_to_render_data_calculator_test.cc +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/util/detections_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" - -namespace mediapipe { - -using ::testing::DoubleNear; - -// Error tolerance for pixels, distances, etc. -static constexpr double kErrorTolerance = 1e-5; - -void VerifyRenderAnnotationColorThickness( - const RenderAnnotation& annotation, - const DetectionsToRenderDataCalculatorOptions& options) { - EXPECT_THAT(annotation.color(), EqualsProto(options.color())); - EXPECT_EQ(annotation.thickness(), options.thickness()); -} - -LocationData CreateLocationData(int32 xmin, int32 ymin, int32 width, - int32 height) { - LocationData location_data; - location_data.set_format(LocationData::BOUNDING_BOX); - location_data.mutable_bounding_box()->set_xmin(xmin); - location_data.mutable_bounding_box()->set_ymin(ymin); - location_data.mutable_bounding_box()->set_width(width); - location_data.mutable_bounding_box()->set_height(height); - return location_data; -} - -LocationData CreateRelativeLocationData(double xmin, double ymin, double width, - double height) { - LocationData location_data; - location_data.set_format(LocationData::RELATIVE_BOUNDING_BOX); - location_data.mutable_relative_bounding_box()->set_xmin(xmin); - location_data.mutable_relative_bounding_box()->set_ymin(ymin); - location_data.mutable_relative_bounding_box()->set_width(width); - location_data.mutable_relative_bounding_box()->set_height(height); - return location_data; -} - -Detection CreateDetection(const std::vector& labels, - const std::vector& label_ids, - const std::vector& scores, - const LocationData& location_data, - const std::string& feature_tag) { - Detection detection; - for (const auto& label : labels) { - detection.add_label(label); - } - for (const auto& label_id : label_ids) { - detection.add_label_id(label_id); - } - for (const auto& score : scores) { - detection.add_score(score); - } - *(detection.mutable_location_data()) = location_data; - detection.set_feature_tag(feature_tag); - return detection; -} - -TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionList) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTION_LIST:detection_list" - output_stream: "RENDER_DATA:render_data" - )pb")); - - LocationData location_data = CreateLocationData(100, 200, 300, 400); - auto detections(absl::make_unique()); - *(detections->add_detection()) = - CreateDetection({"label1"}, {}, {0.3}, location_data, "feature_tag"); - - runner.MutableInputs() - ->Tag("DETECTION_LIST") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("RENDER_DATA").packets; - ASSERT_EQ(1, output.size()); - const auto& actual = output[0].Get(); - EXPECT_EQ(actual.render_annotations_size(), 3); - // Labels - EXPECT_EQ(actual.render_annotations(0).text().display_text(), "label1,0.3,"); - // Feature tag - EXPECT_EQ(actual.render_annotations(1).text().display_text(), "feature_tag"); - // Location data - EXPECT_EQ(actual.render_annotations(2).rectangle().left(), 100); - EXPECT_EQ(actual.render_annotations(2).rectangle().right(), 100 + 300); - EXPECT_EQ(actual.render_annotations(2).rectangle().top(), 200); - EXPECT_EQ(actual.render_annotations(2).rectangle().bottom(), 200 + 400); -} - -TEST(DetectionsToRenderDataCalculatorTest, OnlyDetecctionVector) { - CalculatorRunner runner{ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "RENDER_DATA:render_data" - )pb")}; - - LocationData location_data = CreateLocationData(100, 200, 300, 400); - auto detections(absl::make_unique>()); - detections->push_back( - CreateDetection({"label1"}, {}, {0.3}, location_data, "feature_tag")); - - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = - runner.Outputs().Tag("RENDER_DATA").packets; - ASSERT_EQ(1, output.size()); - const auto& actual = output[0].Get(); - EXPECT_EQ(actual.render_annotations_size(), 3); - // Labels - EXPECT_EQ(actual.render_annotations(0).text().display_text(), "label1,0.3,"); - // Feature tag - EXPECT_EQ(actual.render_annotations(1).text().display_text(), "feature_tag"); - // Location data - EXPECT_EQ(actual.render_annotations(2).rectangle().left(), 100); - EXPECT_EQ(actual.render_annotations(2).rectangle().right(), 100 + 300); - EXPECT_EQ(actual.render_annotations(2).rectangle().top(), 200); - EXPECT_EQ(actual.render_annotations(2).rectangle().bottom(), 200 + 400); -} - -TEST(DetectionsToRenderDataCalculatorTest, BothDetecctionListAndVector) { - CalculatorRunner runner{ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTION_LIST:detection_list" - input_stream: "DETECTIONS:detections" - output_stream: "RENDER_DATA:render_data" - )pb")}; - - LocationData location_data1 = CreateLocationData(100, 200, 300, 400); - auto detection_list(absl::make_unique()); - *(detection_list->add_detection()) = - CreateDetection({"label1"}, {}, {0.3}, location_data1, "feature_tag1"); - runner.MutableInputs() - ->Tag("DETECTION_LIST") - .packets.push_back( - Adopt(detection_list.release()).At(Timestamp::PostStream())); - - LocationData location_data2 = CreateLocationData(600, 700, 800, 900); - auto detections(absl::make_unique>()); - detections->push_back( - CreateDetection({"label2"}, {}, {0.6}, location_data2, "feature_tag2")); - runner.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& actual = - runner.Outputs().Tag("RENDER_DATA").packets; - ASSERT_EQ(1, actual.size()); - // Check the feature tag for item from detection list. - EXPECT_EQ( - actual[0].Get().render_annotations(1).text().display_text(), - "feature_tag1"); - // Check the feature tag for item from detection vector. - EXPECT_EQ( - actual[0].Get().render_annotations(4).text().display_text(), - "feature_tag2"); -} - -TEST(DetectionsToRenderDataCalculatorTest, ProduceEmptyPacket) { - // Check when produce_empty_packet is false. - CalculatorRunner runner1{ - ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTION_LIST:detection_list" - input_stream: "DETECTIONS:detections" - output_stream: "RENDER_DATA:render_data" - options { - [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { - produce_empty_packet: false - } - } - )pb")}; - - auto detection_list1(absl::make_unique()); - runner1.MutableInputs() - ->Tag("DETECTION_LIST") - .packets.push_back( - Adopt(detection_list1.release()).At(Timestamp::PostStream())); - - auto detections1(absl::make_unique>()); - runner1.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections1.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner1.Run()) << "Calculator execution failed."; - const std::vector& exact1 = - runner1.Outputs().Tag("RENDER_DATA").packets; - ASSERT_EQ(0, exact1.size()); - - // Check when produce_empty_packet is true. - CalculatorRunner runner2{ - ParseTextProtoOrDie(R"pb( - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTION_LIST:detection_list" - input_stream: "DETECTIONS:detections" - output_stream: "RENDER_DATA:render_data" - options { - [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { - produce_empty_packet: true - } - } - )pb")}; - - auto detection_list2(absl::make_unique()); - runner2.MutableInputs() - ->Tag("DETECTION_LIST") - .packets.push_back( - Adopt(detection_list2.release()).At(Timestamp::PostStream())); - - auto detections2(absl::make_unique>()); - runner2.MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back( - Adopt(detections2.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner2.Run()) << "Calculator execution failed."; - const std::vector& exact2 = - runner2.Outputs().Tag("RENDER_DATA").packets; - ASSERT_EQ(1, exact2.size()); - EXPECT_EQ(exact2[0].Get().render_annotations_size(), 0); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/detections_to_timed_box_list_calculator.cc b/mediapipe/calculators/util/detections_to_timed_box_list_calculator.cc deleted file mode 100644 index 4b4742b18..000000000 --- a/mediapipe/calculators/util/detections_to_timed_box_list_calculator.cc +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/tracking/box_tracker.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kDetectionListTag[] = "DETECTION_LIST"; -constexpr char kBoxesTag[] = "BOXES"; - -} // namespace - -// A calculator that converts Detection proto to TimedBoxList proto for -// tracking. -// -// Please note that only Location Data formats of RELATIVE_BOUNDING_BOX are -// supported. -// -// Example config: -// node { -// calculator: "DetectionsToTimedBoxListCalculator" -// input_stream: "DETECTIONS:detections" -// output_stream: "BOXES:boxes" -// } -class DetectionsToTimedBoxListCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kDetectionListTag) || - cc->Inputs().HasTag(kDetectionsTag)) - << "None of the input streams are provided."; - if (cc->Inputs().HasTag(kDetectionListTag)) { - cc->Inputs().Tag(kDetectionListTag).Set(); - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - cc->Inputs().Tag(kDetectionsTag).Set>(); - } - cc->Outputs().Tag(kBoxesTag).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - absl::Status Process(CalculatorContext* cc) override; - - private: - void ConvertDetectionToTimedBox(const Detection& detection, - TimedBoxProto* box, CalculatorContext* cc); -}; -REGISTER_CALCULATOR(DetectionsToTimedBoxListCalculator); - -absl::Status DetectionsToTimedBoxListCalculator::Process( - CalculatorContext* cc) { - auto output_timed_box_list = absl::make_unique(); - - if (cc->Inputs().HasTag(kDetectionListTag)) { - const auto& detection_list = - cc->Inputs().Tag(kDetectionListTag).Get(); - for (const auto& detection : detection_list.detection()) { - TimedBoxProto* box = output_timed_box_list->add_box(); - ConvertDetectionToTimedBox(detection, box, cc); - } - } - if (cc->Inputs().HasTag(kDetectionsTag)) { - const auto& detections = - cc->Inputs().Tag(kDetectionsTag).Get>(); - for (const auto& detection : detections) { - TimedBoxProto* box = output_timed_box_list->add_box(); - ConvertDetectionToTimedBox(detection, box, cc); - } - } - - cc->Outputs().Tag(kBoxesTag).Add(output_timed_box_list.release(), - cc->InputTimestamp()); - return absl::OkStatus(); -} - -void DetectionsToTimedBoxListCalculator::ConvertDetectionToTimedBox( - const Detection& detection, TimedBoxProto* box, CalculatorContext* cc) { - const auto& relative_bounding_box = - detection.location_data().relative_bounding_box(); - box->set_left(relative_bounding_box.xmin()); - box->set_right(relative_bounding_box.xmin() + relative_bounding_box.width()); - box->set_top(relative_bounding_box.ymin()); - box->set_bottom(relative_bounding_box.ymin() + - relative_bounding_box.height()); - box->set_id(detection.detection_id()); - box->set_time_msec(cc->InputTimestamp().Microseconds() / 1000); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/filter_collection_calculator.cc b/mediapipe/calculators/util/filter_collection_calculator.cc deleted file mode 100644 index ab361f450..000000000 --- a/mediapipe/calculators/util/filter_collection_calculator.cc +++ /dev/null @@ -1,48 +0,0 @@ - -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/filter_collection_calculator.h" - -#include - -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/integral_types.h" - -namespace mediapipe { - -typedef FilterCollectionCalculator> - FilterUInt64CollectionCalculator; -REGISTER_CALCULATOR(FilterUInt64CollectionCalculator); - -typedef FilterCollectionCalculator> - FilterNormalizedRectCollectionCalculator; -REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator); - -typedef FilterCollectionCalculator> - FilterLandmarkListCollectionCalculator; -REGISTER_CALCULATOR(FilterLandmarkListCollectionCalculator); - -typedef FilterCollectionCalculator< - std::vector<::mediapipe::NormalizedLandmarkList>> - FilterNormalizedLandmarkListCollectionCalculator; -REGISTER_CALCULATOR(FilterNormalizedLandmarkListCollectionCalculator); - -typedef FilterCollectionCalculator> - FilterClassificationListCollectionCalculator; -REGISTER_CALCULATOR(FilterClassificationListCollectionCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/filter_collection_calculator.h b/mediapipe/calculators/util/filter_collection_calculator.h deleted file mode 100644 index 60a6255c9..000000000 --- a/mediapipe/calculators/util/filter_collection_calculator.h +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ - -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -// A calculator that gates elements of an input collection based on -// corresponding boolean values of the "CONDITION" vector. If there is no input -// collection or "CONDITION" vector, the calculator forwards timestamp bounds -// for downstream calculators. If the "CONDITION" vector has false values for -// all elements of the input collection, the calculator outputs a packet -// containing an empty collection. -// Example usage: -// node { -// calculator: "FilterCollectionCalculator" -// input_stream: "ITERABLE:input_collection" -// input_stream: "CONDITION:condition_vector" -// output_stream: "ITERABLE:output_collection" -// } -// This calculator is able to handle collections of copyable types T. -template -class FilterCollectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("ITERABLE")); - RET_CHECK(cc->Inputs().HasTag("CONDITION")); - RET_CHECK(cc->Outputs().HasTag("ITERABLE")); - - cc->Inputs().Tag("ITERABLE").Set(); - cc->Inputs().Tag("CONDITION").Set>(); - - cc->Outputs().Tag("ITERABLE").Set(); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (cc->Inputs().Tag("ITERABLE").IsEmpty()) { - return absl::OkStatus(); - } - if (cc->Inputs().Tag("CONDITION").IsEmpty()) { - return absl::OkStatus(); - } - - const std::vector& filter_by = - cc->Inputs().Tag("CONDITION").Get>(); - - return FilterCollection( - std::is_copy_constructible(), cc, - filter_by); - } - - template - absl::Status FilterCollection(std::true_type, CalculatorContext* cc, - const std::vector& filter_by) { - const IterableU& input = cc->Inputs().Tag("ITERABLE").Get(); - if (input.size() != filter_by.size()) { - return absl::InternalError(absl::StrCat( - "Input vector size: ", input.size(), - " doesn't mach condition vector size: ", filter_by.size())); - } - - auto output = absl::make_unique(); - for (int i = 0; i < input.size(); ++i) { - if (filter_by[i]) { - output->push_back(input[i]); - } - } - cc->Outputs().Tag("ITERABLE").Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - } - - template - absl::Status FilterCollection(std::false_type, CalculatorContext* cc, - const std::vector& filter_by) { - return absl::InternalError("Cannot copy input collection to filter it."); - } -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/from_image_calculator.cc b/mediapipe/calculators/util/from_image_calculator.cc deleted file mode 100644 index 7484d9257..000000000 --- a/mediapipe/calculators/util/from_image_calculator.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { - -namespace { -constexpr char kImageFrameTag[] = "IMAGE_CPU"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kImageTag[] = "IMAGE"; -} // namespace - -// A calculator for converting the unified image container into -// legacy MediaPipe datatypes. -// -// Inputs: -// IMAGE: An Image containing input image. -// -// Output: -// One of the following two tags: -// IMAGE_CPU: An ImageFrame containing output image. -// IMAGE_GPU: A GpuBuffer containing output image. -// -// Note: -// Data is automatically transferred to/from the CPU or GPU -// depending on output type. -// -class FromImageCalculator : public CalculatorBase { - public: - FromImageCalculator() = default; - ~FromImageCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - - bool gpu_output_ = false; - bool gpu_initialized_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(FromImageCalculator); - -absl::Status FromImageCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kImageTag).Set(); - - bool gpu_output = false; - - if (cc->Outputs().HasTag(kImageFrameTag) && - cc->Outputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("Cannot have multiple outputs."); - } - - if (cc->Outputs().HasTag(kGpuBufferTag)) { -#if !MEDIAPIPE_DISABLE_GPU - cc->Outputs().Tag(kGpuBufferTag).Set(); - gpu_output = true; -#else - RET_CHECK_FAIL() << "GPU is disabled. Cannot use IMAGE_GPU stream."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - if (cc->Outputs().HasTag(kImageFrameTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); - } - - if (gpu_output) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status FromImageCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Outputs().HasTag(kGpuBufferTag)) { - gpu_output_ = true; - } - - if (gpu_output_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif - } // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status FromImageCalculator::Process(CalculatorContext* cc) { - if (gpu_output_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([&cc]() -> absl::Status { - auto& input = cc->Inputs().Tag(kImageTag).Get(); - // Unwrap texture pointer; shallow copy. - auto output = - std::make_unique(input.GetGpuBuffer()); - cc->Outputs() - .Tag(kGpuBufferTag) - .Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - // The input Image. - auto& input = cc->Inputs().Tag(kImageTag).Get(); - // Make a copy of the input packet to co-own the input Image. - Packet* packet_copy_ptr = new Packet(cc->Inputs().Tag(kImageTag).Value()); - // Create an output ImageFrame that points to the same pixel data as the - // input Image and also owns the packet copy. As a result, the output - // ImageFrame indirectly co-owns the input Image. This ensures a correct - // life span of the shared pixel data. - std::unique_ptr output = - std::make_unique( - input.image_format(), input.width(), input.height(), input.step(), - const_cast(input.GetImageFrameSharedPtr()->PixelData()), - [packet_copy_ptr](uint8*) { delete packet_copy_ptr; }); - cc->Outputs() - .Tag(kImageFrameTag) - .Add(output.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status FromImageCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/labels_to_render_data_calculator.cc b/mediapipe/calculators/util/labels_to_render_data_calculator.cc deleted file mode 100644 index 099bdc7e6..000000000 --- a/mediapipe/calculators/util/labels_to_render_data_calculator.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "mediapipe/calculators/util/labels_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" - -namespace mediapipe { - -constexpr float kFontHeightScale = 1.25f; - -// A calculator takes in pairs of labels and scores or classifications, outputs -// generates render data. Either both "LABELS" and "SCORES" or "CLASSIFICATIONS" -// must be present. -// -// Usage example: -// node { -// calculator: "LabelsToRenderDataCalculator" -// input_stream: "LABELS:labels" -// input_stream: "SCORES:scores" -// output_stream: "VIDEO_PRESTREAM:video_header" -// options { -// [LabelsToRenderDataCalculatorOptions.ext] { -// color { r: 255 g: 0 b: 0 } -// color { r: 0 g: 255 b: 0 } -// color { r: 0 g: 0 b: 255 } -// thickness: 2.0 -// font_height_px: 20 -// max_num_labels: 3 -// font_face: 1 -// location: TOP_LEFT -// } -// } -// } -class LabelsToRenderDataCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - LabelsToRenderDataCalculatorOptions options_; - int num_colors_ = 0; - int video_width_ = 0; - int video_height_ = 0; - int label_height_px_ = 0; - int label_left_px_ = 0; -}; -REGISTER_CALCULATOR(LabelsToRenderDataCalculator); - -absl::Status LabelsToRenderDataCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag("CLASSIFICATIONS")) { - cc->Inputs().Tag("CLASSIFICATIONS").Set(); - } else { - RET_CHECK(cc->Inputs().HasTag("LABELS")) - << "Must provide input stream \"LABELS\""; - cc->Inputs().Tag("LABELS").Set>(); - if (cc->Inputs().HasTag("SCORES")) { - cc->Inputs().Tag("SCORES").Set>(); - } - } - if (cc->Inputs().HasTag("VIDEO_PRESTREAM")) { - cc->Inputs().Tag("VIDEO_PRESTREAM").Set(); - } - cc->Outputs().Tag("RENDER_DATA").Set(); - return absl::OkStatus(); -} - -absl::Status LabelsToRenderDataCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - options_ = cc->Options(); - num_colors_ = options_.color_size(); - label_height_px_ = std::ceil(options_.font_height_px() * kFontHeightScale); - return absl::OkStatus(); -} - -absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag("VIDEO_PRESTREAM") && - cc->InputTimestamp() == Timestamp::PreStream()) { - const VideoHeader& video_header = - cc->Inputs().Tag("VIDEO_PRESTREAM").Get(); - video_width_ = video_header.width; - video_height_ = video_header.height; - return absl::OkStatus(); - } else { - CHECK_EQ(options_.location(), LabelsToRenderDataCalculatorOptions::TOP_LEFT) - << "Only TOP_LEFT is supported without VIDEO_PRESTREAM."; - } - - std::vector labels; - std::vector scores; - if (cc->Inputs().HasTag("CLASSIFICATIONS")) { - const ClassificationList& classifications = - cc->Inputs().Tag("CLASSIFICATIONS").Get(); - labels.resize(classifications.classification_size()); - scores.resize(classifications.classification_size()); - for (int i = 0; i < classifications.classification_size(); ++i) { - if (options_.use_display_name()) { - labels[i] = classifications.classification(i).display_name(); - } else { - labels[i] = classifications.classification(i).label(); - } - scores[i] = classifications.classification(i).score(); - } - } else { - const std::vector& label_vector = - cc->Inputs().Tag("LABELS").Get>(); - labels.resize(label_vector.size()); - for (int i = 0; i < label_vector.size(); ++i) { - labels[i] = label_vector[i]; - } - - if (cc->Inputs().HasTag("SCORES")) { - std::vector score_vector = - cc->Inputs().Tag("SCORES").Get>(); - CHECK_EQ(label_vector.size(), score_vector.size()); - scores.resize(label_vector.size()); - for (int i = 0; i < label_vector.size(); ++i) { - scores[i] = score_vector[i]; - } - } - } - - RenderData render_data; - int num_label = std::min((int)labels.size(), options_.max_num_labels()); - int label_baseline_px = options_.vertical_offset_px(); - if (options_.location() == LabelsToRenderDataCalculatorOptions::TOP_LEFT) { - label_baseline_px += label_height_px_; - } else if (options_.location() == - LabelsToRenderDataCalculatorOptions::BOTTOM_LEFT) { - label_baseline_px += video_height_ - label_height_px_ * (num_label - 1); - } - label_left_px_ = options_.horizontal_offset_px(); - for (int i = 0; i < num_label; ++i) { - auto* label_annotation = render_data.add_render_annotations(); - label_annotation->set_thickness(options_.thickness()); - if (num_colors_ > 0) { - *(label_annotation->mutable_color()) = options_.color(i % num_colors_); - } else { - label_annotation->mutable_color()->set_r(255); - label_annotation->mutable_color()->set_g(0); - label_annotation->mutable_color()->set_b(0); - } - - auto* text = label_annotation->mutable_text(); - std::string display_text = labels[i]; - if (cc->Inputs().HasTag("SCORES")) { - absl::StrAppend(&display_text, ":", scores[i]); - } - text->set_display_text(display_text); - text->set_font_height(options_.font_height_px()); - text->set_left(label_left_px_); - text->set_baseline(label_baseline_px + i * label_height_px_); - text->set_font_face(options_.font_face()); - } - cc->Outputs() - .Tag("RENDER_DATA") - .AddPacket(MakePacket(render_data).At(cc->InputTimestamp())); - - return absl::OkStatus(); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/util/labels_to_render_data_calculator.proto b/mediapipe/calculators/util/labels_to_render_data_calculator.proto deleted file mode 100644 index c5012ce85..000000000 --- a/mediapipe/calculators/util/labels_to_render_data_calculator.proto +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -message LabelsToRenderDataCalculatorOptions { - extend CalculatorOptions { - optional LabelsToRenderDataCalculatorOptions ext = 271660364; - } - - // Colors for drawing the label(s). - repeated Color color = 1; - - // Thickness for drawing the label(s). - optional double thickness = 2 [default = 2]; - - // The font height in absolute pixels. - optional int32 font_height_px = 3 [default = 50]; - - // The offset of the starting text in horizontal direction in absolute pixels. - optional int32 horizontal_offset_px = 7 [default = 0]; - // The offset of the starting text in vertical direction in absolute pixels. - optional int32 vertical_offset_px = 8 [default = 0]; - - // The maximum number of labels to display. - optional int32 max_num_labels = 4 [default = 1]; - - // Specifies the font for the text. Font must be one of the following from - // OpenCV: - // cv::FONT_HERSHEY_SIMPLEX (0) - // cv::FONT_HERSHEY_PLAIN (1) - // cv::FONT_HERSHEY_DUPLEX (2) - // cv::FONT_HERSHEY_COMPLEX (3) - // cv::FONT_HERSHEY_TRIPLEX (4) - // cv::FONT_HERSHEY_COMPLEX_SMALL (5) - // cv::FONT_HERSHEY_SCRIPT_SIMPLEX (6) - // cv::FONT_HERSHEY_SCRIPT_COMPLEX (7) - optional int32 font_face = 5 [default = 0]; - - // Label location. - enum Location { - TOP_LEFT = 0; - BOTTOM_LEFT = 1; - } - optional Location location = 6 [default = TOP_LEFT]; - - // Uses Classification.display_name field instead of Classification.label. - optional bool use_display_name = 9 [default = false]; -} diff --git a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc b/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc deleted file mode 100644 index d3c7a6453..000000000 --- a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kLandmarksTag[] = "LANDMARKS"; -constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING"; - -} // namespace - -// Adjusts landmark locations on a letterboxed image to the corresponding -// locations on the same image with the letterbox removed. This is useful to map -// the landmarks inferred from a letterboxed image, for example, output of -// the ImageTransformationCalculator when the scale mode is FIT, back to the -// corresponding input image before letterboxing. -// -// Input: -// LANDMARKS: A NormalizedLandmarkList representing landmarks on an -// letterboxed image. -// -// LETTERBOX_PADDING: An std::array representing the letterbox -// padding from the 4 sides ([left, top, right, bottom]) of the letterboxed -// image, normalized to [0.f, 1.f] by the letterboxed image dimensions. -// -// Output: -// LANDMARKS: An NormalizedLandmarkList proto representing landmarks with -// their locations adjusted to the letterbox-removed (non-padded) image. -// -// Usage example: -// node { -// calculator: "LandmarkLetterboxRemovalCalculator" -// input_stream: "LANDMARKS:landmarks" -// input_stream: "LETTERBOX_PADDING:letterbox_padding" -// output_stream: "LANDMARKS:adjusted_landmarks" -// } -// -// node { -// calculator: "LandmarkLetterboxRemovalCalculator" -// input_stream: "LANDMARKS:0:landmarks_0" -// input_stream: "LANDMARKS:1:landmarks_1" -// input_stream: "LETTERBOX_PADDING:letterbox_padding" -// output_stream: "LANDMARKS:0:adjusted_landmarks_0" -// output_stream: "LANDMARKS:1:adjusted_landmarks_1" -// } -class LandmarkLetterboxRemovalCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().HasTag(kLetterboxPaddingTag)) - << "Missing one or more input streams."; - - RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag), - cc->Outputs().NumEntries(kLandmarksTag)) - << "Same number of input and output landmarks is required."; - - for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag); - id != cc->Inputs().EndId(kLandmarksTag); ++id) { - cc->Inputs().Get(id).Set(); - } - cc->Inputs().Tag(kLetterboxPaddingTag).Set>(); - - for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag); - id != cc->Outputs().EndId(kLandmarksTag); ++id) { - cc->Outputs().Get(id).Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto& letterbox_padding = - cc->Inputs().Tag(kLetterboxPaddingTag).Get>(); - const float left = letterbox_padding[0]; - const float top = letterbox_padding[1]; - const float left_and_right = letterbox_padding[0] + letterbox_padding[2]; - const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3]; - - CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag); - CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag); - // Number of inputs and outpus is the same according to the contract. - for (; input_id != cc->Inputs().EndId(kLandmarksTag); - ++input_id, ++output_id) { - const auto& input_packet = cc->Inputs().Get(input_id); - if (input_packet.IsEmpty()) { - continue; - } - - const NormalizedLandmarkList& input_landmarks = - input_packet.Get(); - NormalizedLandmarkList output_landmarks; - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = input_landmarks.landmark(i); - NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); - const float new_x = (landmark.x() - left) / (1.0f - left_and_right); - const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom); - const float new_z = - landmark.z() / (1.0f - left_and_right); // Scale Z coordinate as X. - *new_landmark = landmark; - new_landmark->set_x(new_x); - new_landmark->set_y(new_y); - new_landmark->set_z(new_z); - } - - cc->Outputs().Get(output_id).AddPacket( - MakePacket(output_landmarks) - .At(cc->InputTimestamp())); - } - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(LandmarkLetterboxRemovalCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc b/mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc deleted file mode 100644 index 556d5673d..000000000 --- a/mediapipe/calculators/util/landmark_letterbox_removal_calculator_test.cc +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/tool/validate_type.h" - -namespace mediapipe { - -NormalizedLandmark CreateLandmark(float x, float y) { - NormalizedLandmark landmark; - landmark.set_x(x); - landmark.set_y(y); - return landmark; -} - -CalculatorGraphConfig::Node GetDefaultNode() { - return ParseTextProtoOrDie(R"pb( - calculator: "LandmarkLetterboxRemovalCalculator" - input_stream: "LANDMARKS:landmarks" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "LANDMARKS:adjusted_landmarks" - )pb"); -} - -TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingLeftRight) { - CalculatorRunner runner(GetDefaultNode()); - - auto landmarks = absl::make_unique(); - *landmarks->add_landmark() = CreateLandmark(0.5f, 0.5f); - *landmarks->add_landmark() = CreateLandmark(0.2f, 0.2f); - *landmarks->add_landmark() = CreateLandmark(0.7f, 0.7f); - runner.MutableInputs() - ->Tag("LANDMARKS") - .packets.push_back( - Adopt(landmarks.release()).At(Timestamp::PostStream())); - - auto padding = absl::make_unique>( - std::array{0.2f, 0.f, 0.3f, 0.f}); - runner.MutableInputs() - ->Tag("LETTERBOX_PADDING") - .packets.push_back(Adopt(padding.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("LANDMARKS").packets; - ASSERT_EQ(1, output.size()); - const auto& output_landmarks = output[0].Get(); - - EXPECT_EQ(output_landmarks.landmark_size(), 3); - - EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.6f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.5f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.0f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.2f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(1.0f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(0.7f, 1e-5)); -} - -TEST(LandmarkLetterboxRemovalCalculatorTest, PaddingTopBottom) { - CalculatorRunner runner(GetDefaultNode()); - - auto landmarks = absl::make_unique(); - NormalizedLandmark* landmark = landmarks->add_landmark(); - *landmark = CreateLandmark(0.5f, 0.5f); - landmark = landmarks->add_landmark(); - *landmark = CreateLandmark(0.2f, 0.2f); - landmark = landmarks->add_landmark(); - *landmark = CreateLandmark(0.7f, 0.7f); - runner.MutableInputs() - ->Tag("LANDMARKS") - .packets.push_back( - Adopt(landmarks.release()).At(Timestamp::PostStream())); - - auto padding = absl::make_unique>( - std::array{0.0f, 0.2f, 0.0f, 0.3f}); - runner.MutableInputs() - ->Tag("LETTERBOX_PADDING") - .packets.push_back(Adopt(padding.release()).At(Timestamp::PostStream())); - - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output = runner.Outputs().Tag("LANDMARKS").packets; - ASSERT_EQ(1, output.size()); - const auto& output_landmarks = output[0].Get(); - - EXPECT_EQ(output_landmarks.landmark_size(), 3); - - EXPECT_THAT(output_landmarks.landmark(0).x(), testing::FloatNear(0.5f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(0).y(), testing::FloatNear(0.6f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(1).x(), testing::FloatNear(0.2f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(1).y(), testing::FloatNear(0.0f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(2).x(), testing::FloatNear(0.7f, 1e-5)); - EXPECT_THAT(output_landmarks.landmark(2).y(), testing::FloatNear(1.0f, 1e-5)); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmark_projection_calculator.cc b/mediapipe/calculators/util/landmark_projection_calculator.cc deleted file mode 100644 index e27edea66..000000000 --- a/mediapipe/calculators/util/landmark_projection_calculator.cc +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "mediapipe/calculators/util/landmark_projection_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kRectTag[] = "NORM_RECT"; -constexpr char kProjectionMatrix[] = "PROJECTION_MATRIX"; - -} // namespace - -// Projects normalized landmarks to its original coordinates. -// Input: -// NORM_LANDMARKS - NormalizedLandmarkList -// Represents landmarks in a normalized rectangle if NORM_RECT is specified -// or landmarks that should be projected using PROJECTION_MATRIX if -// specified. (Prefer using PROJECTION_MATRIX as it eliminates need of -// letterbox removal step.) -// NORM_RECT - NormalizedRect -// Represents a normalized rectangle in image coordinates and results in -// landmarks with their locations adjusted to the image. -// PROJECTION_MATRIX - std::array -// A 4x4 row-major-order matrix that maps landmarks' locations from one -// coordinate system to another. In this case from the coordinate system of -// the normalized region of interest to the coordinate system of the image. -// -// Note: either NORM_RECT or PROJECTION_MATRIX has to be specified. -// Note: landmark's Z is projected in a custom way - it's scaled by width of -// the normalized region of interest used during landmarks detection. -// -// Output: -// NORM_LANDMARKS - NormalizedLandmarkList -// Landmarks with their locations adjusted according to the inputs. -// -// Usage example: -// node { -// calculator: "LandmarkProjectionCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// input_stream: "NORM_RECT:rect" -// output_stream: "NORM_LANDMARKS:projected_landmarks" -// } -// -// node { -// calculator: "LandmarkProjectionCalculator" -// input_stream: "NORM_LANDMARKS:0:landmarks_0" -// input_stream: "NORM_LANDMARKS:1:landmarks_1" -// input_stream: "NORM_RECT:rect" -// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0" -// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1" -// } -// -// node { -// calculator: "LandmarkProjectionCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// input_stream: "PROECTION_MATRIX:matrix" -// output_stream: "NORM_LANDMARKS:projected_landmarks" -// } -// -// node { -// calculator: "LandmarkProjectionCalculator" -// input_stream: "NORM_LANDMARKS:0:landmarks_0" -// input_stream: "NORM_LANDMARKS:1:landmarks_1" -// input_stream: "PROECTION_MATRIX:matrix" -// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0" -// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1" -// } -class LandmarkProjectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kLandmarksTag)) - << "Missing NORM_LANDMARKS input."; - - RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag), - cc->Outputs().NumEntries(kLandmarksTag)) - << "Same number of input and output landmarks is required."; - - for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag); - id != cc->Inputs().EndId(kLandmarksTag); ++id) { - cc->Inputs().Get(id).Set(); - } - RET_CHECK(cc->Inputs().HasTag(kRectTag) ^ - cc->Inputs().HasTag(kProjectionMatrix)) - << "Either NORM_RECT or PROJECTION_MATRIX must be specified."; - if (cc->Inputs().HasTag(kRectTag)) { - cc->Inputs().Tag(kRectTag).Set(); - } else { - cc->Inputs().Tag(kProjectionMatrix).Set>(); - } - - for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag); - id != cc->Outputs().EndId(kLandmarksTag); ++id) { - cc->Outputs().Get(id).Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); - } - - static void ProjectXY(const NormalizedLandmark& lm, - const std::array& matrix, - NormalizedLandmark* out) { - out->set_x(lm.x() * matrix[0] + lm.y() * matrix[1] + lm.z() * matrix[2] + - matrix[3]); - out->set_y(lm.x() * matrix[4] + lm.y() * matrix[5] + lm.z() * matrix[6] + - matrix[7]); - } - - /** - * Landmark's Z scale is equal to a relative (to image) width of region of - * interest used during detection. To calculate based on matrix: - * 1. Project (0,0) --- (1,0) segment using matrix. - * 2. Calculate length of the projected segment. - */ - static float CalculateZScale(const std::array& matrix) { - NormalizedLandmark a; - a.set_x(0.0f); - a.set_y(0.0f); - NormalizedLandmark b; - b.set_x(1.0f); - b.set_y(0.0f); - NormalizedLandmark a_projected; - ProjectXY(a, matrix, &a_projected); - NormalizedLandmark b_projected; - ProjectXY(b, matrix, &b_projected); - return std::sqrt(std::pow(b_projected.x() - a_projected.x(), 2) + - std::pow(b_projected.y() - a_projected.y(), 2)); - } - - absl::Status Process(CalculatorContext* cc) override { - std::function - project_fn; - if (cc->Inputs().HasTag(kRectTag)) { - if (cc->Inputs().Tag(kRectTag).IsEmpty()) { - return absl::OkStatus(); - } - const auto& input_rect = cc->Inputs().Tag(kRectTag).Get(); - const auto& options = - cc->Options(); - project_fn = [&input_rect, &options](const NormalizedLandmark& landmark, - NormalizedLandmark* new_landmark) { - // TODO: fix projection or deprecate (current projection - // calculations are incorrect for general case). - const float x = landmark.x() - 0.5f; - const float y = landmark.y() - 0.5f; - const float angle = - options.ignore_rotation() ? 0 : input_rect.rotation(); - float new_x = std::cos(angle) * x - std::sin(angle) * y; - float new_y = std::sin(angle) * x + std::cos(angle) * y; - - new_x = new_x * input_rect.width() + input_rect.x_center(); - new_y = new_y * input_rect.height() + input_rect.y_center(); - const float new_z = - landmark.z() * input_rect.width(); // Scale Z coordinate as X. - - *new_landmark = landmark; - new_landmark->set_x(new_x); - new_landmark->set_y(new_y); - new_landmark->set_z(new_z); - }; - } else if (cc->Inputs().HasTag(kProjectionMatrix)) { - if (cc->Inputs().Tag(kProjectionMatrix).IsEmpty()) { - return absl::OkStatus(); - } - const auto& project_mat = - cc->Inputs().Tag(kProjectionMatrix).Get>(); - const float z_scale = CalculateZScale(project_mat); - project_fn = [&project_mat, z_scale](const NormalizedLandmark& lm, - NormalizedLandmark* new_landmark) { - *new_landmark = lm; - ProjectXY(lm, project_mat, new_landmark); - new_landmark->set_z(z_scale * lm.z()); - }; - } else { - return absl::InternalError("Either rect or matrix must be specified."); - } - - CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag); - CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag); - // Number of inputs and outpus is the same according to the contract. - for (; input_id != cc->Inputs().EndId(kLandmarksTag); - ++input_id, ++output_id) { - const auto& input_packet = cc->Inputs().Get(input_id); - if (input_packet.IsEmpty()) { - continue; - } - - const auto& input_landmarks = input_packet.Get(); - NormalizedLandmarkList output_landmarks; - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = input_landmarks.landmark(i); - NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); - project_fn(landmark, new_landmark); - } - - cc->Outputs().Get(output_id).AddPacket( - MakePacket(std::move(output_landmarks)) - .At(cc->InputTimestamp())); - } - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(LandmarkProjectionCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmark_projection_calculator.proto b/mediapipe/calculators/util/landmark_projection_calculator.proto deleted file mode 100644 index 221adbe00..000000000 --- a/mediapipe/calculators/util/landmark_projection_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LandmarkProjectionCalculatorOptions { - extend CalculatorOptions { - optional LandmarkProjectionCalculatorOptions ext = 263371892; - } - - // Ignore the rotation field of rect proto for projection. - optional bool ignore_rotation = 1 [default = false]; -} diff --git a/mediapipe/calculators/util/landmark_projection_calculator_test.cc b/mediapipe/calculators/util/landmark_projection_calculator_test.cc deleted file mode 100644 index b15bb0f0c..000000000 --- a/mediapipe/calculators/util/landmark_projection_calculator_test.cc +++ /dev/null @@ -1,240 +0,0 @@ -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -absl::StatusOr RunCalculator( - mediapipe::NormalizedLandmarkList input, mediapipe::NormalizedRect rect) { - mediapipe::CalculatorRunner runner( - ParseTextProtoOrDie(R"pb( - calculator: "LandmarkProjectionCalculator" - input_stream: "NORM_LANDMARKS:landmarks" - input_stream: "NORM_RECT:rect" - output_stream: "NORM_LANDMARKS:projected_landmarks" - )pb")); - runner.MutableInputs() - ->Tag("NORM_LANDMARKS") - .packets.push_back( - MakePacket(std::move(input)) - .At(Timestamp(1))); - runner.MutableInputs() - ->Tag("NORM_RECT") - .packets.push_back(MakePacket(std::move(rect)) - .At(Timestamp(1))); - - MP_RETURN_IF_ERROR(runner.Run()); - const auto& output_packets = runner.Outputs().Tag("NORM_LANDMARKS").packets; - RET_CHECK_EQ(output_packets.size(), 1); - return output_packets[0].Get(); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithDefaultRect) { - mediapipe::NormalizedLandmarkList landmarks = - ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - )pb"); - mediapipe::NormalizedRect rect = - ParseTextProtoOrDie( - R"pb( - x_center: 0.5, - y_center: 0.5, - width: 1.0, - height: 1.0, - rotation: 0.0 - )pb"); - - auto status_or_result = RunCalculator(std::move(landmarks), std::move(rect)); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT( - status_or_result.value(), - EqualsProto(ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - )pb"))); -} - -mediapipe::NormalizedRect GetCroppedRect() { - return ParseTextProtoOrDie( - R"pb( - x_center: 0.5, y_center: 0.5, width: 0.5, height: 2, rotation: 0.0 - )pb"); -} - -mediapipe::NormalizedLandmarkList GetCroppedRectTestInput() { - return ParseTextProtoOrDie(R"pb( - landmark { x: 1.0, y: 1.0, z: -0.5 } - )pb"); -} - -mediapipe::NormalizedLandmarkList GetCroppedRectTestExpectedResult() { - return ParseTextProtoOrDie(R"pb( - landmark { x: 0.75, y: 1.5, z: -0.25 } - )pb"); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithCroppedRect) { - auto status_or_result = - RunCalculator(GetCroppedRectTestInput(), GetCroppedRect()); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT(status_or_result.value(), - EqualsProto(GetCroppedRectTestExpectedResult())); -} - -absl::StatusOr RunCalculator( - mediapipe::NormalizedLandmarkList input, std::array matrix) { - mediapipe::CalculatorRunner runner( - ParseTextProtoOrDie(R"pb( - calculator: "LandmarkProjectionCalculator" - input_stream: "NORM_LANDMARKS:landmarks" - input_stream: "PROJECTION_MATRIX:matrix" - output_stream: "NORM_LANDMARKS:projected_landmarks" - )pb")); - runner.MutableInputs() - ->Tag("NORM_LANDMARKS") - .packets.push_back( - MakePacket(std::move(input)) - .At(Timestamp(1))); - runner.MutableInputs() - ->Tag("PROJECTION_MATRIX") - .packets.push_back(MakePacket>(std::move(matrix)) - .At(Timestamp(1))); - - MP_RETURN_IF_ERROR(runner.Run()); - const auto& output_packets = runner.Outputs().Tag("NORM_LANDMARKS").packets; - RET_CHECK_EQ(output_packets.size(), 1); - return output_packets[0].Get(); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithIdentityMatrix) { - mediapipe::NormalizedLandmarkList landmarks = - ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - )pb"); - // clang-format off - std::array matrix = { - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, - }; - // clang-format on - - auto status_or_result = - RunCalculator(std::move(landmarks), std::move(matrix)); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT( - status_or_result.value(), - EqualsProto(ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - )pb"))); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithCroppedRectMatrix) { - constexpr int kRectWidth = 1280; - constexpr int kRectHeight = 720; - auto roi = GetRoi(kRectWidth, kRectHeight, GetCroppedRect()); - std::array matrix; - GetRotatedSubRectToRectTransformMatrix(roi, kRectWidth, kRectHeight, - /*flip_horizontaly=*/false, &matrix); - auto status_or_result = RunCalculator(GetCroppedRectTestInput(), matrix); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT(status_or_result.value(), - EqualsProto(GetCroppedRectTestExpectedResult())); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithScaleMatrix) { - mediapipe::NormalizedLandmarkList landmarks = - ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - landmark { x: 5, y: 6, z: 7 } - )pb"); - // clang-format off - std::array matrix = { - 10.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 100.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, - }; - // clang-format on - - auto status_or_result = - RunCalculator(std::move(landmarks), std::move(matrix)); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT( - status_or_result.value(), - EqualsProto(ParseTextProtoOrDie(R"pb( - landmark { x: 100, y: 2000, z: -5 } - landmark { x: 50, y: 600, z: 70 } - )pb"))); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithTranslateMatrix) { - mediapipe::NormalizedLandmarkList landmarks = - ParseTextProtoOrDie(R"pb( - landmark { x: 10, y: 20, z: -0.5 } - )pb"); - // clang-format off - std::array matrix = { - 1.0f, 0.0f, 0.0f, 1.0f, - 0.0f, 1.0f, 0.0f, 2.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, - }; - // clang-format on - - auto status_or_result = - RunCalculator(std::move(landmarks), std::move(matrix)); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT( - status_or_result.value(), - EqualsProto(ParseTextProtoOrDie(R"pb( - landmark { x: 11, y: 22, z: -0.5 } - )pb"))); -} - -TEST(LandmarkProjectionCalculatorTest, ProjectingWithRotationMatrix) { - mediapipe::NormalizedLandmarkList landmarks = - ParseTextProtoOrDie(R"pb( - landmark { x: 4, y: 0, z: -0.5 } - )pb"); - // clang-format off - // 90 degrees rotation matrix - std::array matrix = { - 0.0f, -1.0f, 0.0f, 0.0f, - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, - }; - // clang-format on - - auto status_or_result = - RunCalculator(std::move(landmarks), std::move(matrix)); - MP_ASSERT_OK(status_or_result); - - EXPECT_THAT( - status_or_result.value(), - EqualsProto(ParseTextProtoOrDie(R"pb( - landmark { x: 0, y: 4, z: -0.5 } - )pb"))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmark_visibility_calculator.cc b/mediapipe/calculators/util/landmark_visibility_calculator.cc deleted file mode 100644 index f22d2ac57..000000000 --- a/mediapipe/calculators/util/landmark_visibility_calculator.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kVisibilityTag[] = "VISIBILITY"; - -} // namespace - -// A calculator to extract visibility from the landmark. -// -// Inputs: -// NORM_LANDMARKS: A NormalizedLandmarkList with only a single landmark to -// take visibility from. It's a list and not single landmark as -// split/concatenate calculators work with lists. -// -// Outputs: -// VISIBILITY: Float visibility of the given landmark. -// -// Example config: -// node { -// calculator: "LandmarkVisibilityCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// output_stream: "VISIBILITY:visibility" -// } -// -class LandmarkVisibilityCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(LandmarkVisibilityCalculator); - -absl::Status LandmarkVisibilityCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kNormalizedLandmarksTag).Set(); - cc->Outputs().Tag(kVisibilityTag).Set(); - - return absl::OkStatus(); -} - -absl::Status LandmarkVisibilityCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); -} - -absl::Status LandmarkVisibilityCalculator::Process(CalculatorContext* cc) { - // Check that landmark is not empty. - // Don't emit an empty packet for this timestamp. - if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& landmarks = - cc->Inputs().Tag(kNormalizedLandmarksTag).Get(); - RET_CHECK_EQ(landmarks.landmark_size(), 1); - float visibility = landmarks.landmark(0).visibility(); - - cc->Outputs() - .Tag(kVisibilityTag) - .AddPacket(MakePacket(visibility).At(cc->InputTimestamp())); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc b/mediapipe/calculators/util/landmarks_smoothing_calculator.cc deleted file mode 100644 index 6673816e7..000000000 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc +++ /dev/null @@ -1,480 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/algorithm/container.h" -#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/util/filtering/one_euro_filter.h" -#include "mediapipe/util/filtering/relative_velocity_filter.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kLandmarksTag[] = "LANDMARKS"; -constexpr char kImageSizeTag[] = "IMAGE_SIZE"; -constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI"; -constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS"; -constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS"; - -using mediapipe::OneEuroFilter; -using mediapipe::RelativeVelocityFilter; - -void NormalizedLandmarksToLandmarks( - const NormalizedLandmarkList& norm_landmarks, const int image_width, - const int image_height, LandmarkList* landmarks) { - for (int i = 0; i < norm_landmarks.landmark_size(); ++i) { - const auto& norm_landmark = norm_landmarks.landmark(i); - - auto* landmark = landmarks->add_landmark(); - landmark->set_x(norm_landmark.x() * image_width); - landmark->set_y(norm_landmark.y() * image_height); - // Scale Z the same way as X (using image width). - landmark->set_z(norm_landmark.z() * image_width); - landmark->set_visibility(norm_landmark.visibility()); - landmark->set_presence(norm_landmark.presence()); - } -} - -void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks, - const int image_width, - const int image_height, - NormalizedLandmarkList* norm_landmarks) { - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const auto& landmark = landmarks.landmark(i); - - auto* norm_landmark = norm_landmarks->add_landmark(); - norm_landmark->set_x(landmark.x() / image_width); - norm_landmark->set_y(landmark.y() / image_height); - // Scale Z the same way as X (using image width). - norm_landmark->set_z(landmark.z() / image_width); - norm_landmark->set_visibility(landmark.visibility()); - norm_landmark->set_presence(landmark.presence()); - } -} - -// Estimate object scale to use its inverse value as velocity scale for -// RelativeVelocityFilter. If value will be too small (less than -// `options_.min_allowed_object_scale`) smoothing will be disabled and -// landmarks will be returned as is. -// Object scale is calculated as average between bounding box width and height -// with sides parallel to axis. -float GetObjectScale(const LandmarkList& landmarks) { - const auto& lm_minmax_x = absl::c_minmax_element( - landmarks.landmark(), - [](const auto& a, const auto& b) { return a.x() < b.x(); }); - const float x_min = lm_minmax_x.first->x(); - const float x_max = lm_minmax_x.second->x(); - - const auto& lm_minmax_y = absl::c_minmax_element( - landmarks.landmark(), - [](const auto& a, const auto& b) { return a.y() < b.y(); }); - const float y_min = lm_minmax_y.first->y(); - const float y_max = lm_minmax_y.second->y(); - - const float object_width = x_max - x_min; - const float object_height = y_max - y_min; - - return (object_width + object_height) / 2.0f; -} - -float GetObjectScale(const NormalizedRect& roi, const int image_width, - const int image_height) { - const float object_width = roi.width() * image_width; - const float object_height = roi.height() * image_height; - - return (object_width + object_height) / 2.0f; -} - -float GetObjectScale(const Rect& roi) { - return (roi.width() + roi.height()) / 2.0f; -} - -// Abstract class for various landmarks filters. -class LandmarksFilter { - public: - virtual ~LandmarksFilter() = default; - - virtual absl::Status Reset() { return absl::OkStatus(); } - - virtual absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - const absl::optional object_scale_opt, - LandmarkList* out_landmarks) = 0; -}; - -// Returns landmarks as is without smoothing. -class NoFilter : public LandmarksFilter { - public: - absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - const absl::optional object_scale_opt, - LandmarkList* out_landmarks) override { - *out_landmarks = in_landmarks; - return absl::OkStatus(); - } -}; - -// Please check RelativeVelocityFilter documentation for details. -class VelocityFilter : public LandmarksFilter { - public: - VelocityFilter(int window_size, float velocity_scale, - float min_allowed_object_scale, bool disable_value_scaling) - : window_size_(window_size), - velocity_scale_(velocity_scale), - min_allowed_object_scale_(min_allowed_object_scale), - disable_value_scaling_(disable_value_scaling) {} - - absl::Status Reset() override { - x_filters_.clear(); - y_filters_.clear(); - z_filters_.clear(); - return absl::OkStatus(); - } - - absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - const absl::optional object_scale_opt, - LandmarkList* out_landmarks) override { - // Get value scale as inverse value of the object scale. - // If value is too small smoothing will be disabled and landmarks will be - // returned as is. - float value_scale = 1.0f; - if (!disable_value_scaling_) { - const float object_scale = - object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks); - if (object_scale < min_allowed_object_scale_) { - *out_landmarks = in_landmarks; - return absl::OkStatus(); - } - value_scale = 1.0f / object_scale; - } - - // Initialize filters once. - MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size())); - - // Filter landmarks. Every axis of every landmark is filtered separately. - for (int i = 0; i < in_landmarks.landmark_size(); ++i) { - const auto& in_landmark = in_landmarks.landmark(i); - - auto* out_landmark = out_landmarks->add_landmark(); - *out_landmark = in_landmark; - out_landmark->set_x( - x_filters_[i].Apply(timestamp, value_scale, in_landmark.x())); - out_landmark->set_y( - y_filters_[i].Apply(timestamp, value_scale, in_landmark.y())); - out_landmark->set_z( - z_filters_[i].Apply(timestamp, value_scale, in_landmark.z())); - } - - return absl::OkStatus(); - } - - private: - // Initializes filters for the first time or after Reset. If initialized then - // check the size. - absl::Status InitializeFiltersIfEmpty(const int n_landmarks) { - if (!x_filters_.empty()) { - RET_CHECK_EQ(x_filters_.size(), n_landmarks); - RET_CHECK_EQ(y_filters_.size(), n_landmarks); - RET_CHECK_EQ(z_filters_.size(), n_landmarks); - return absl::OkStatus(); - } - - x_filters_.resize(n_landmarks, - RelativeVelocityFilter(window_size_, velocity_scale_)); - y_filters_.resize(n_landmarks, - RelativeVelocityFilter(window_size_, velocity_scale_)); - z_filters_.resize(n_landmarks, - RelativeVelocityFilter(window_size_, velocity_scale_)); - - return absl::OkStatus(); - } - - int window_size_; - float velocity_scale_; - float min_allowed_object_scale_; - bool disable_value_scaling_; - - std::vector x_filters_; - std::vector y_filters_; - std::vector z_filters_; -}; - -// Please check OneEuroFilter documentation for details. -class OneEuroFilterImpl : public LandmarksFilter { - public: - OneEuroFilterImpl(double frequency, double min_cutoff, double beta, - double derivate_cutoff, float min_allowed_object_scale, - bool disable_value_scaling) - : frequency_(frequency), - min_cutoff_(min_cutoff), - beta_(beta), - derivate_cutoff_(derivate_cutoff), - min_allowed_object_scale_(min_allowed_object_scale), - disable_value_scaling_(disable_value_scaling) {} - - absl::Status Reset() override { - x_filters_.clear(); - y_filters_.clear(); - z_filters_.clear(); - return absl::OkStatus(); - } - - absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - const absl::optional object_scale_opt, - LandmarkList* out_landmarks) override { - // Initialize filters once. - MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size())); - - // Get value scale as inverse value of the object scale. - // If value is too small smoothing will be disabled and landmarks will be - // returned as is. - float value_scale = 1.0f; - if (!disable_value_scaling_) { - const float object_scale = - object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks); - if (object_scale < min_allowed_object_scale_) { - *out_landmarks = in_landmarks; - return absl::OkStatus(); - } - value_scale = 1.0f / object_scale; - } - - // Filter landmarks. Every axis of every landmark is filtered separately. - for (int i = 0; i < in_landmarks.landmark_size(); ++i) { - const auto& in_landmark = in_landmarks.landmark(i); - - auto* out_landmark = out_landmarks->add_landmark(); - *out_landmark = in_landmark; - out_landmark->set_x( - x_filters_[i].Apply(timestamp, value_scale, in_landmark.x())); - out_landmark->set_y( - y_filters_[i].Apply(timestamp, value_scale, in_landmark.y())); - out_landmark->set_z( - z_filters_[i].Apply(timestamp, value_scale, in_landmark.z())); - } - - return absl::OkStatus(); - } - - private: - // Initializes filters for the first time or after Reset. If initialized then - // check the size. - absl::Status InitializeFiltersIfEmpty(const int n_landmarks) { - if (!x_filters_.empty()) { - RET_CHECK_EQ(x_filters_.size(), n_landmarks); - RET_CHECK_EQ(y_filters_.size(), n_landmarks); - RET_CHECK_EQ(z_filters_.size(), n_landmarks); - return absl::OkStatus(); - } - - for (int i = 0; i < n_landmarks; ++i) { - x_filters_.push_back( - OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_)); - y_filters_.push_back( - OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_)); - z_filters_.push_back( - OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_)); - } - - return absl::OkStatus(); - } - - double frequency_; - double min_cutoff_; - double beta_; - double derivate_cutoff_; - double min_allowed_object_scale_; - bool disable_value_scaling_; - - std::vector x_filters_; - std::vector y_filters_; - std::vector z_filters_; -}; - -} // namespace - -// A calculator to smooth landmarks over time. -// -// Inputs: -// NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth. -// IMAGE_SIZE: A std::pair represention of image width and height. -// Required to perform all computations in absolute coordinates to avoid any -// influence of normalized values. -// OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of -// input landmarks) used to determine the object scale for some of the -// filters. If not provided - object scale will be calculated from -// landmarks. -// -// Outputs: -// NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks. -// -// Example config: -// node { -// calculator: "LandmarksSmoothingCalculator" -// input_stream: "NORM_LANDMARKS:pose_landmarks" -// input_stream: "IMAGE_SIZE:image_size" -// input_stream: "OBJECT_SCALE_ROI:roi" -// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered" -// options: { -// [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { -// velocity_filter: { -// window_size: 5 -// velocity_scale: 10.0 -// } -// } -// } -// } -// -class LandmarksSmoothingCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - std::unique_ptr landmarks_filter_; -}; -REGISTER_CALCULATOR(LandmarksSmoothingCalculator); - -absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) { - cc->Inputs().Tag(kNormalizedLandmarksTag).Set(); - cc->Inputs().Tag(kImageSizeTag).Set>(); - cc->Outputs() - .Tag(kNormalizedFilteredLandmarksTag) - .Set(); - - if (cc->Inputs().HasTag(kObjectScaleRoiTag)) { - cc->Inputs().Tag(kObjectScaleRoiTag).Set(); - } - } else { - cc->Inputs().Tag(kLandmarksTag).Set(); - cc->Outputs().Tag(kFilteredLandmarksTag).Set(); - - if (cc->Inputs().HasTag(kObjectScaleRoiTag)) { - cc->Inputs().Tag(kObjectScaleRoiTag).Set(); - } - } - - return absl::OkStatus(); -} - -absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - // Pick landmarks filter. - const auto& options = cc->Options(); - if (options.has_no_filter()) { - landmarks_filter_ = absl::make_unique(); - } else if (options.has_velocity_filter()) { - landmarks_filter_ = absl::make_unique( - options.velocity_filter().window_size(), - options.velocity_filter().velocity_scale(), - options.velocity_filter().min_allowed_object_scale(), - options.velocity_filter().disable_value_scaling()); - } else if (options.has_one_euro_filter()) { - landmarks_filter_ = absl::make_unique( - options.one_euro_filter().frequency(), - options.one_euro_filter().min_cutoff(), - options.one_euro_filter().beta(), - options.one_euro_filter().derivate_cutoff(), - options.one_euro_filter().min_allowed_object_scale(), - options.one_euro_filter().disable_value_scaling()); - } else { - RET_CHECK_FAIL() - << "Landmarks filter is either not specified or not supported"; - } - - return absl::OkStatus(); -} - -absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) { - // Check that landmarks are not empty and reset the filter if so. - // Don't emit an empty packet for this timestamp. - if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) && - cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) || - (cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().Tag(kLandmarksTag).IsEmpty())) { - MP_RETURN_IF_ERROR(landmarks_filter_->Reset()); - return absl::OkStatus(); - } - - const auto& timestamp = - absl::Microseconds(cc->InputTimestamp().Microseconds()); - - if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) { - const auto& in_norm_landmarks = - cc->Inputs().Tag(kNormalizedLandmarksTag).Get(); - - int image_width; - int image_height; - std::tie(image_width, image_height) = - cc->Inputs().Tag(kImageSizeTag).Get>(); - - absl::optional object_scale; - if (cc->Inputs().HasTag(kObjectScaleRoiTag) && - !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) { - auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get(); - object_scale = GetObjectScale(roi, image_width, image_height); - } - - auto in_landmarks = absl::make_unique(); - NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height, - in_landmarks.get()); - - auto out_landmarks = absl::make_unique(); - MP_RETURN_IF_ERROR(landmarks_filter_->Apply( - *in_landmarks, timestamp, object_scale, out_landmarks.get())); - - auto out_norm_landmarks = absl::make_unique(); - LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height, - out_norm_landmarks.get()); - - cc->Outputs() - .Tag(kNormalizedFilteredLandmarksTag) - .Add(out_norm_landmarks.release(), cc->InputTimestamp()); - } else { - const auto& in_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - - absl::optional object_scale; - if (cc->Inputs().HasTag(kObjectScaleRoiTag) && - !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) { - auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get(); - object_scale = GetObjectScale(roi); - } - - auto out_landmarks = absl::make_unique(); - MP_RETURN_IF_ERROR(landmarks_filter_->Apply( - in_landmarks, timestamp, object_scale, out_landmarks.get())); - - cc->Outputs() - .Tag(kFilteredLandmarksTag) - .Add(out_landmarks.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator.proto b/mediapipe/calculators/util/landmarks_smoothing_calculator.proto deleted file mode 100644 index 017facb30..000000000 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator.proto +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator_options.proto"; - -message LandmarksSmoothingCalculatorOptions { - extend CalculatorOptions { - optional LandmarksSmoothingCalculatorOptions ext = 325671429; - } - - // Default behaviour and fast way to disable smoothing. - message NoFilter {} - - message VelocityFilter { - // Number of value changes to keep over time. - // Higher value adds to lag and to stability. - optional int32 window_size = 1 [default = 5]; - - // Scale to apply to the velocity calculated over the given window. With - // higher velocity `low pass filter` weights new values higher. - // Lower value adds to lag and to stability. - optional float velocity_scale = 2 [default = 10.0]; - - // If calculated object scale is less than given value smoothing will be - // disabled and landmarks will be returned as is. - optional float min_allowed_object_scale = 3 [default = 1e-6]; - - // Disable value scaling based on object size and use `1.0` instead. - // If not disabled, value scale is calculated as inverse value of object - // size. Object size is calculated as maximum side of rectangular bounding - // box of the object in XY plane. - optional bool disable_value_scaling = 4 [default = false]; - } - - // For the details of the filter implementation and the procedure of its - // configuration please check http://cristal.univ-lille.fr/~casiez/1euro/ - message OneEuroFilter { - // Frequency of incomming frames defined in frames per seconds. Used only if - // can't be calculated from provided events (e.g. on the very first frame). - optional float frequency = 1 [default = 30.0]; - - // Minimum cutoff frequency. Start by tuning this parameter while keeping - // `beta = 0` to reduce jittering to the desired level. 1Hz (the default - // value) is a good starting point. - optional float min_cutoff = 2 [default = 1.0]; - - // Cutoff slope. After `min_cutoff` is configured, start increasing `beta` - // value to reduce the lag introduced by the `min_cutoff`. Find the desired - // balance between jittering and lag. - optional float beta = 3 [default = 0.0]; - - // Cutoff frequency for derivate. It is set to 1Hz in the original - // algorithm, but can be tuned to further smooth the speed (i.e. derivate) - // on the object. - optional float derivate_cutoff = 4 [default = 1.0]; - - // If calculated object scale is less than given value smoothing will be - // disabled and landmarks will be returned as is. - optional float min_allowed_object_scale = 5 [default = 1e-6]; - - // Disable value scaling based on object size and use `1.0` instead. - // If not disabled, value scale is calculated as inverse value of object - // size. Object size is calculated as maximum side of rectangular bounding - // box of the object in XY plane. - optional bool disable_value_scaling = 6 [default = false]; - } - - oneof filter_options { - NoFilter no_filter = 1; - VelocityFilter velocity_filter = 2; - OneEuroFilter one_euro_filter = 3; - } -} diff --git a/mediapipe/calculators/util/landmarks_to_detection_calculator.cc b/mediapipe/calculators/util/landmarks_to_detection_calculator.cc deleted file mode 100644 index ffa359877..000000000 --- a/mediapipe/calculators/util/landmarks_to_detection_calculator.cc +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/util/landmarks_to_detection_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kDetectionTag[] = "DETECTION"; -constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS"; - -Detection ConvertLandmarksToDetection(const NormalizedLandmarkList& landmarks) { - Detection detection; - LocationData* location_data = detection.mutable_location_data(); - - float x_min = std::numeric_limits::max(); - float x_max = std::numeric_limits::min(); - float y_min = std::numeric_limits::max(); - float y_max = std::numeric_limits::min(); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = landmarks.landmark(i); - x_min = std::min(x_min, landmark.x()); - x_max = std::max(x_max, landmark.x()); - y_min = std::min(y_min, landmark.y()); - y_max = std::max(y_max, landmark.y()); - - auto keypoint = location_data->add_relative_keypoints(); - keypoint->set_x(landmark.x()); - keypoint->set_y(landmark.y()); - } - - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - LocationData::RelativeBoundingBox* relative_bbox = - location_data->mutable_relative_bounding_box(); - - relative_bbox->set_xmin(x_min); - relative_bbox->set_ymin(y_min); - relative_bbox->set_width(x_max - x_min); - relative_bbox->set_height(y_max - y_min); - - return detection; -} - -} // namespace - -// Converts NormalizedLandmark to Detection proto. A relative bounding box will -// be created containing all landmarks exactly. A calculator option is provided -// to specify a subset of landmarks for creating the detection. -// -// Input: -// NOMR_LANDMARKS: A NormalizedLandmarkList proto. -// -// Output: -// DETECTION: A Detection proto. -// -// Example config: -// node { -// calculator: "LandmarksToDetectionCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// output_stream: "DETECTION:detections" -// } -class LandmarksToDetectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - ::mediapipe::LandmarksToDetectionCalculatorOptions options_; -}; -REGISTER_CALCULATOR(LandmarksToDetectionCalculator); - -absl::Status LandmarksToDetectionCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag)); - RET_CHECK(cc->Outputs().HasTag(kDetectionTag)); - // TODO: Also support converting Landmark to Detection. - cc->Inputs().Tag(kNormalizedLandmarksTag).Set(); - cc->Outputs().Tag(kDetectionTag).Set(); - - return absl::OkStatus(); -} - -absl::Status LandmarksToDetectionCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options<::mediapipe::LandmarksToDetectionCalculatorOptions>(); - return absl::OkStatus(); -} - -absl::Status LandmarksToDetectionCalculator::Process(CalculatorContext* cc) { - const auto& landmarks = - cc->Inputs().Tag(kNormalizedLandmarksTag).Get(); - RET_CHECK_GT(landmarks.landmark_size(), 0) - << "Input landmark vector is empty."; - - auto detection = absl::make_unique(); - if (options_.selected_landmark_indices_size()) { - NormalizedLandmarkList subset_landmarks; - for (int i = 0; i < options_.selected_landmark_indices_size(); ++i) { - RET_CHECK_LT(options_.selected_landmark_indices(i), - landmarks.landmark_size()) - << "Index of landmark subset is out of range."; - *subset_landmarks.add_landmark() = - landmarks.landmark(options_.selected_landmark_indices(i)); - } - *detection = ConvertLandmarksToDetection(subset_landmarks); - } else { - *detection = ConvertLandmarksToDetection(landmarks); - } - cc->Outputs() - .Tag(kDetectionTag) - .Add(detection.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_to_detection_calculator.proto b/mediapipe/calculators/util/landmarks_to_detection_calculator.proto deleted file mode 100644 index b5a563669..000000000 --- a/mediapipe/calculators/util/landmarks_to_detection_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LandmarksToDetectionCalculatorOptions { - extend CalculatorOptions { - optional LandmarksToDetectionCalculatorOptions ext = 260199669; - } - - // A subset of indices to be included when creating the detection. - repeated int32 selected_landmark_indices = 1; -} diff --git a/mediapipe/calculators/util/landmarks_to_floats_calculator.cc b/mediapipe/calculators/util/landmarks_to_floats_calculator.cc deleted file mode 100644 index fe8dd3ab1..000000000 --- a/mediapipe/calculators/util/landmarks_to_floats_calculator.cc +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "Eigen/Core" -#include "mediapipe/calculators/util/landmarks_to_floats_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kFloatsTag[] = "FLOATS"; -constexpr char kMatrixTag[] = "MATRIX"; - -} // namespace - -// Converts a vector of landmarks to a vector of floats or a matrix. -// Input: -// NORM_LANDMARKS: A NormalizedLandmarkList proto. -// -// Output: -// FLOATS(optional): A vector of floats from flattened landmarks. -// MATRIX(optional): A matrix of floats of the landmarks. -// -// Usage example: -// node { -// calculator: "LandmarksToFloatsCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// output_stream: "MATRIX:landmark_matrix" -// } -class LandmarksToFloatsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kLandmarksTag).Set(); - RET_CHECK(cc->Outputs().HasTag(kFloatsTag) || - cc->Outputs().HasTag(kMatrixTag)); - if (cc->Outputs().HasTag(kFloatsTag)) { - cc->Outputs().Tag(kFloatsTag).Set>(); - } - if (cc->Outputs().HasTag(kMatrixTag)) { - cc->Outputs().Tag(kMatrixTag).Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - const auto& options = - cc->Options<::mediapipe::LandmarksToFloatsCalculatorOptions>(); - num_dimensions_ = options.num_dimensions(); - // Currently number of dimensions must be within [1, 3]. - RET_CHECK_GE(num_dimensions_, 1); - RET_CHECK_LE(num_dimensions_, 3); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - // Only process if there's input landmarks. - if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& input_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - - if (cc->Outputs().HasTag(kFloatsTag)) { - auto output_floats = absl::make_unique>(); - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = input_landmarks.landmark(i); - output_floats->emplace_back(landmark.x()); - if (num_dimensions_ > 1) { - output_floats->emplace_back(landmark.y()); - } - if (num_dimensions_ > 2) { - output_floats->emplace_back(landmark.z()); - } - } - - cc->Outputs() - .Tag(kFloatsTag) - .Add(output_floats.release(), cc->InputTimestamp()); - } else { - auto output_matrix = absl::make_unique(); - output_matrix->setZero(num_dimensions_, input_landmarks.landmark_size()); - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - (*output_matrix)(0, i) = input_landmarks.landmark(i).x(); - if (num_dimensions_ > 1) { - (*output_matrix)(1, i) = input_landmarks.landmark(i).y(); - } - if (num_dimensions_ > 2) { - (*output_matrix)(2, i) = input_landmarks.landmark(i).z(); - } - } - cc->Outputs() - .Tag(kMatrixTag) - .Add(output_matrix.release(), cc->InputTimestamp()); - } - return absl::OkStatus(); - } - - private: - int num_dimensions_ = 0; -}; -REGISTER_CALCULATOR(LandmarksToFloatsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_to_floats_calculator.proto b/mediapipe/calculators/util/landmarks_to_floats_calculator.proto deleted file mode 100644 index 310251e75..000000000 --- a/mediapipe/calculators/util/landmarks_to_floats_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LandmarksToFloatsCalculatorOptions { - extend CalculatorOptions { - optional LandmarksToFloatsCalculatorOptions ext = 274035660; - } - - // Number of dimensions to convert. Must within [1, 3]. - optional int32 num_dimensions = 1 [default = 2]; -} diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc deleted file mode 100644 index f2cec3ae3..000000000 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.h" - -#include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" -namespace mediapipe { - -namespace { - -constexpr char kLandmarksTag[] = "LANDMARKS"; -constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kRenderScaleTag[] = "RENDER_SCALE"; -constexpr char kRenderDataTag[] = "RENDER_DATA"; -constexpr char kLandmarkLabel[] = "KEYPOINT"; - -inline Color DefaultMinDepthLineColor() { - Color color; - color.set_r(0); - color.set_g(0); - color.set_b(0); - return color; -} - -inline Color DefaultMaxDepthLineColor() { - Color color; - color.set_r(255); - color.set_g(255); - color.set_b(255); - return color; -} - -inline Color MixColors(const Color& color1, const Color& color2, - float color1_weight) { - Color color; - color.set_r(static_cast(color1.r() * color1_weight + - color2.r() * (1.f - color1_weight))); - color.set_g(static_cast(color1.g() * color1_weight + - color2.g() * (1.f - color1_weight))); - color.set_b(static_cast(color1.b() * color1_weight + - color2.b() * (1.f - color1_weight))); - return color; -} - -inline void SetColor(RenderAnnotation* annotation, const Color& color) { - annotation->mutable_color()->set_r(color.r()); - annotation->mutable_color()->set_g(color.g()); - annotation->mutable_color()->set_b(color.b()); -} - -// Remap x from range [lo hi] to range [0 1] then multiply by scale. -inline float Remap(float x, float lo, float hi, float scale) { - return (x - lo) / (hi - lo + 1e-6) * scale; -} - -template -inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min, - float* z_max) { - *z_min = std::numeric_limits::max(); - *z_max = std::numeric_limits::min(); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const LandmarkType& landmark = landmarks.landmark(i); - *z_min = std::min(landmark.z(), *z_min); - *z_max = std::max(landmark.z(), *z_max); - } -} - -template -bool IsLandmarkVisibileAndPresent(const LandmarkType& landmark, - bool utilize_visibility, - float visibility_threshold, - bool utilize_presence, - float presence_threshold) { - if (utilize_visibility && landmark.has_visibility() && - landmark.visibility() < visibility_threshold) { - return false; - } - if (utilize_presence && landmark.has_presence() && - landmark.presence() < presence_threshold) { - return false; - } - return true; -} - -void SetColorSizeValueFromZ(float z, float z_min, float z_max, - RenderAnnotation* render_annotation, - float min_depth_circle_thickness, - float max_depth_circle_thickness) { - const int color_value = 255 - static_cast(Remap(z, z_min, z_max, 255)); - ::mediapipe::Color color; - color.set_r(color_value); - color.set_g(color_value); - color.set_b(color_value); - SetColor(render_annotation, color); - const float scale = max_depth_circle_thickness - min_depth_circle_thickness; - const int thickness = static_cast( - min_depth_circle_thickness + (1.f - Remap(z, z_min, z_max, 1)) * scale); - render_annotation->set_thickness(thickness); -} - -template -void AddConnectionToRenderData(const LandmarkType& start, - const LandmarkType& end, - const Color& color_start, const Color& color_end, - float thickness, bool normalized, - RenderData* render_data) { - auto* connection_annotation = render_data->add_render_annotations(); - RenderAnnotation::GradientLine* line = - connection_annotation->mutable_gradient_line(); - line->set_x_start(start.x()); - line->set_y_start(start.y()); - line->set_x_end(end.x()); - line->set_y_end(end.y()); - line->set_normalized(normalized); - line->mutable_color1()->set_r(color_start.r()); - line->mutable_color1()->set_g(color_start.g()); - line->mutable_color1()->set_b(color_start.b()); - line->mutable_color2()->set_r(color_end.r()); - line->mutable_color2()->set_g(color_end.g()); - line->mutable_color2()->set_b(color_end.b()); - - connection_annotation->set_thickness(thickness); -} - -template -void AddConnectionsWithDepth(const LandmarkListType& landmarks, - const std::vector& landmark_connections, - bool utilize_visibility, - float visibility_threshold, bool utilize_presence, - float presence_threshold, float thickness, - bool normalized, float min_z, float max_z, - const Color& min_depth_line_color, - const Color& max_depth_line_color, - RenderData* render_data) { - for (int i = 0; i < landmark_connections.size(); i += 2) { - const auto& ld0 = landmarks.landmark(landmark_connections[i]); - const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]); - if (!IsLandmarkVisibileAndPresent( - ld0, utilize_visibility, visibility_threshold, utilize_presence, - presence_threshold) || - !IsLandmarkVisibileAndPresent( - ld1, utilize_visibility, visibility_threshold, utilize_presence, - presence_threshold)) { - continue; - } - const Color color0 = MixColors(min_depth_line_color, max_depth_line_color, - Remap(ld0.z(), min_z, max_z, 1.f)); - const Color color1 = MixColors(min_depth_line_color, max_depth_line_color, - Remap(ld1.z(), min_z, max_z, 1.f)); - AddConnectionToRenderData(ld0, ld1, color0, color1, thickness, - normalized, render_data); - } -} - -template -void AddConnectionToRenderData(const LandmarkType& start, - const LandmarkType& end, - const Color& connection_color, float thickness, - bool normalized, RenderData* render_data) { - auto* connection_annotation = render_data->add_render_annotations(); - RenderAnnotation::Line* line = connection_annotation->mutable_line(); - line->set_x_start(start.x()); - line->set_y_start(start.y()); - line->set_x_end(end.x()); - line->set_y_end(end.y()); - line->set_normalized(normalized); - SetColor(connection_annotation, connection_color); - connection_annotation->set_thickness(thickness); -} - -template -void AddConnections(const LandmarkListType& landmarks, - const std::vector& landmark_connections, - bool utilize_visibility, float visibility_threshold, - bool utilize_presence, float presence_threshold, - const Color& connection_color, float thickness, - bool normalized, RenderData* render_data) { - for (int i = 0; i < landmark_connections.size(); i += 2) { - const auto& ld0 = landmarks.landmark(landmark_connections[i]); - const auto& ld1 = landmarks.landmark(landmark_connections[i + 1]); - if (!IsLandmarkVisibileAndPresent( - ld0, utilize_visibility, visibility_threshold, utilize_presence, - presence_threshold) || - !IsLandmarkVisibileAndPresent( - ld1, utilize_visibility, visibility_threshold, utilize_presence, - presence_threshold)) { - continue; - } - AddConnectionToRenderData(ld0, ld1, connection_color, - thickness, normalized, render_data); - } -} - -RenderAnnotation* AddPointRenderData(const Color& landmark_color, - float thickness, RenderData* render_data) { - auto* landmark_data_annotation = render_data->add_render_annotations(); - landmark_data_annotation->set_scene_tag(kLandmarkLabel); - SetColor(landmark_data_annotation, landmark_color); - landmark_data_annotation->set_thickness(thickness); - return landmark_data_annotation; -} - -} // namespace - -absl::Status LandmarksToRenderDataCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kLandmarksTag) || - cc->Inputs().HasTag(kNormLandmarksTag)) - << "None of the input streams are provided."; - RET_CHECK(!(cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().HasTag(kNormLandmarksTag))) - << "Can only one type of landmark can be taken. Either absolute or " - "normalized landmarks."; - - if (cc->Inputs().HasTag(kLandmarksTag)) { - cc->Inputs().Tag(kLandmarksTag).Set(); - } - if (cc->Inputs().HasTag(kNormLandmarksTag)) { - cc->Inputs().Tag(kNormLandmarksTag).Set(); - } - if (cc->Inputs().HasTag(kRenderScaleTag)) { - cc->Inputs().Tag(kRenderScaleTag).Set(); - } - cc->Outputs().Tag(kRenderDataTag).Set(); - return absl::OkStatus(); -} - -absl::Status LandmarksToRenderDataCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - options_ = cc->Options(); - - // Parse landmarks connections to a vector. - RET_CHECK_EQ(options_.landmark_connections_size() % 2, 0) - << "Number of entries in landmark connections must be a multiple of 2"; - - for (int i = 0; i < options_.landmark_connections_size(); ++i) { - landmark_connections_.push_back(options_.landmark_connections(i)); - } - - return absl::OkStatus(); -} - -absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) { - // Check that landmarks are not empty and skip rendering if so. - // Don't emit an empty packet for this timestamp. - if (cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().Tag(kLandmarksTag).IsEmpty()) { - return absl::OkStatus(); - } - if (cc->Inputs().HasTag(kNormLandmarksTag) && - cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) { - return absl::OkStatus(); - } - - auto render_data = absl::make_unique(); - bool visualize_depth = options_.visualize_landmark_depth(); - float z_min = 0.f; - float z_max = 0.f; - - const Color min_depth_line_color = options_.has_min_depth_line_color() - ? options_.min_depth_line_color() - : DefaultMinDepthLineColor(); - const Color max_depth_line_color = options_.has_max_depth_line_color() - ? options_.max_depth_line_color() - : DefaultMaxDepthLineColor(); - - // Apply scale to `thickness` of rendered landmarks and connections to make - // them bigger when object (e.g. pose, hand or face) is closer/bigger and - // snaller when object is further/smaller. - float thickness = options_.thickness(); - if (cc->Inputs().HasTag(kRenderScaleTag)) { - const float render_scale = cc->Inputs().Tag(kRenderScaleTag).Get(); - thickness *= render_scale; - } - - if (cc->Inputs().HasTag(kLandmarksTag)) { - const LandmarkList& landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - if (visualize_depth) { - GetMinMaxZ(landmarks, &z_min, &z_max); - } - // Only change rendering if there are actually z values other than 0. - visualize_depth &= ((z_max - z_min) > 1e-3); - if (visualize_depth) { - AddConnectionsWithDepth( - landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold(), thickness, /*normalized=*/false, z_min, - z_max, min_depth_line_color, max_depth_line_color, render_data.get()); - } else { - AddConnections( - landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold(), options_.connection_color(), thickness, - /*normalized=*/false, render_data.get()); - } - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const Landmark& landmark = landmarks.landmark(i); - - if (!IsLandmarkVisibileAndPresent( - landmark, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold())) { - continue; - } - - auto* landmark_data_render = AddPointRenderData( - options_.landmark_color(), thickness, render_data.get()); - if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, - options_.min_depth_circle_thickness(), - options_.max_depth_circle_thickness()); - } - auto* landmark_data = landmark_data_render->mutable_point(); - landmark_data->set_normalized(false); - landmark_data->set_x(landmark.x()); - landmark_data->set_y(landmark.y()); - } - } - - if (cc->Inputs().HasTag(kNormLandmarksTag)) { - const NormalizedLandmarkList& landmarks = - cc->Inputs().Tag(kNormLandmarksTag).Get(); - if (visualize_depth) { - GetMinMaxZ(landmarks, &z_min, - &z_max); - } - // Only change rendering if there are actually z values other than 0. - visualize_depth &= ((z_max - z_min) > 1e-3); - if (visualize_depth) { - AddConnectionsWithDepth( - landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold(), thickness, /*normalized=*/true, z_min, - z_max, min_depth_line_color, max_depth_line_color, render_data.get()); - } else { - AddConnections( - landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold(), options_.connection_color(), thickness, - /*normalized=*/true, render_data.get()); - } - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = landmarks.landmark(i); - - if (!IsLandmarkVisibileAndPresent( - landmark, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold())) { - continue; - } - - auto* landmark_data_render = AddPointRenderData( - options_.landmark_color(), thickness, render_data.get()); - if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, - options_.min_depth_circle_thickness(), - options_.max_depth_circle_thickness()); - } - auto* landmark_data = landmark_data_render->mutable_point(); - landmark_data->set_normalized(true); - landmark_data->set_x(landmark.x()); - landmark_data->set_y(landmark.y()); - } - } - - cc->Outputs() - .Tag(kRenderDataTag) - .Add(render_data.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(LandmarksToRenderDataCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.h b/mediapipe/calculators/util/landmarks_to_render_data_calculator.h deleted file mode 100644 index 0fbe9700c..000000000 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_RENDER_DATA_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_RENDER_DATA_CALCULATOR_H_ - -#include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" -namespace mediapipe { - -// A calculator that converts Landmark proto to RenderData proto for -// visualization. The input should be LandmarkList proto. It is also possible -// to specify the connections between landmarks. -// -// Example config: -// node { -// calculator: "LandmarksToRenderDataCalculator" -// input_stream: "NORM_LANDMARKS:landmarks" -// output_stream: "RENDER_DATA:render_data" -// options { -// [LandmarksToRenderDataCalculatorOptions.ext] { -// landmark_connections: [0, 1, 1, 2] -// landmark_color { r: 0 g: 255 b: 0 } -// connection_color { r: 0 g: 255 b: 0 } -// thickness: 4.0 -// } -// } -// } -class LandmarksToRenderDataCalculator : public CalculatorBase { - public: - LandmarksToRenderDataCalculator() {} - ~LandmarksToRenderDataCalculator() override {} - LandmarksToRenderDataCalculator(const LandmarksToRenderDataCalculator&) = - delete; - LandmarksToRenderDataCalculator& operator=( - const LandmarksToRenderDataCalculator&) = delete; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - protected: - ::mediapipe::LandmarksToRenderDataCalculatorOptions options_; - std::vector landmark_connections_; -}; - -} // namespace mediapipe -#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_RENDER_DATA_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto b/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto deleted file mode 100644 index 990919540..000000000 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -message LandmarksToRenderDataCalculatorOptions { - extend CalculatorOptions { - optional LandmarksToRenderDataCalculatorOptions ext = 258435389; - } - - // Specifies the landmarks to be connected in the drawing. For example, the - // landmark_connections value of [0, 1, 1, 2] specifies two connections: one - // that connects landmarks with index 0 and 1, and another that connects - // landmarks with index 1 and 2. - repeated int32 landmark_connections = 1; - - // Color of the landmarks. - optional Color landmark_color = 2; - // Color of the connections. - optional Color connection_color = 3; - - // Thickness of the drawing of landmarks and connections. - optional double thickness = 4 [default = 1.0]; - - // Change color and size of rendered landmarks based on its z value. - optional bool visualize_landmark_depth = 5 [default = true]; - - // Use landmarks visibility while rendering landmarks and connections. If - // landmark is not visible, neither it nor adjacent connections will be - // rendered. - optional bool utilize_visibility = 6 [default = false]; - - // Threshold to determine visibility of the landmark. Landmark with visibility - // greater or equal than threshold is considered visible. - optional double visibility_threshold = 7 [default = 0.0]; - - // Use landmarks presence while rendering landmarks and connections. If - // landmark is not present, neither it nor adjacent connections will be - // rendered. - optional bool utilize_presence = 8 [default = false]; - - // Threshold to determine presence of the landmark. Landmark with presence - // greater or equal than threshold is considered present. - optional double presence_threshold = 9 [default = 0.0]; - - // Min thickness of the drawing for landmark circle. - optional double min_depth_circle_thickness = 10 [default = 0.0]; - - // Max thickness of the drawing for landmark circle. - optional double max_depth_circle_thickness = 11 [default = 18.0]; - - // Gradient color for the lines connecting landmarks at the minimum depth. - optional Color min_depth_line_color = 12; - - // Gradient color for the lines connecting landmarks at the maximum depth. - optional Color max_depth_line_color = 13; -} diff --git a/mediapipe/calculators/util/latency.proto b/mediapipe/calculators/util/latency.proto deleted file mode 100644 index 4b122fb19..000000000 --- a/mediapipe/calculators/util/latency.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Proto messages related to latency measurement for Soapbox. -syntax = "proto2"; - -package mediapipe; - -// Contains the latency information for a packet stream in mediapipe. The -// following are provided -// 1. current latency -// 2. running average -// 3. histogram of latencies observed -// 4. cumulative sum of latencies observed -// NextId: 13 -message PacketLatency { - // Reserved tags. - reserved 1, 3 to 6; - - // Current latency (delay in microseconds wrt a reference packet). - optional int64 current_latency_usec = 8; - - // The latency histogram which stores the count recorded for each specified - // interval. - repeated int64 counts = 9; - - // Number of intervals for the latency histogram output. - optional int64 num_intervals = 10 [default = 10]; - - // Size of the histogram intervals (in microseconds). The first interval is - // [0, interval_size_usec). The last interval extends to +inf. - optional int64 interval_size_usec = 11 [default = 10000]; - - // Running average of latencies observed so far. - optional int64 avg_latency_usec = 2; - - // An identifier label for the packet. - optional string label = 7; - - // Cumulative sum of individual packet latencies of all the packets output so - // far. - optional int64 sum_latency_usec = 12; -} diff --git a/mediapipe/calculators/util/local_file_contents_calculator.cc b/mediapipe/calculators/util/local_file_contents_calculator.cc deleted file mode 100644 index 4ad066f69..000000000 --- a/mediapipe/calculators/util/local_file_contents_calculator.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/calculators/util/local_file_contents_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/resource_util.h" - -namespace mediapipe { - -namespace { - -constexpr char kFilePathTag[] = "FILE_PATH"; -constexpr char kContentsTag[] = "CONTENTS"; - -} // namespace - -// The calculator takes the path to the local file as an input side packet and -// outputs the contents of that file. -// -// NOTE: file loading can be batched by providing multiple input/output side -// packets. -// -// Example config: -// node { -// calculator: "LocalFileContentsCalculator" -// input_side_packet: "FILE_PATH:file_path" -// output_side_packet: "CONTENTS:contents" -// } -// -// node { -// calculator: "LocalFileContentsCalculator" -// input_side_packet: "FILE_PATH:0:file_path1" -// input_side_packet: "FILE_PATH:1:file_path2" -// ... -// output_side_packet: "CONTENTS:0:contents1" -// output_side_packet: "CONTENTS:1:contents2" -// ... -// } -class LocalFileContentsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->InputSidePackets().HasTag(kFilePathTag)) - << "Missing PATH input side packet(s)"; - RET_CHECK(cc->OutputSidePackets().HasTag(kContentsTag)) - << "Missing CONTENTS output side packet(s)"; - - RET_CHECK_EQ(cc->InputSidePackets().NumEntries(kFilePathTag), - cc->OutputSidePackets().NumEntries(kContentsTag)) - << "Same number of input streams and output streams is required."; - - for (CollectionItemId id = cc->InputSidePackets().BeginId(kFilePathTag); - id != cc->InputSidePackets().EndId(kFilePathTag); ++id) { - cc->InputSidePackets().Get(id).Set(); - } - - for (CollectionItemId id = cc->OutputSidePackets().BeginId(kContentsTag); - id != cc->OutputSidePackets().EndId(kContentsTag); ++id) { - cc->OutputSidePackets().Get(id).Set(); - } - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - CollectionItemId input_id = cc->InputSidePackets().BeginId(kFilePathTag); - CollectionItemId output_id = cc->OutputSidePackets().BeginId(kContentsTag); - auto options = cc->Options(); - - // Number of inputs and outpus is the same according to the contract. - for (; input_id != cc->InputSidePackets().EndId(kFilePathTag); - ++input_id, ++output_id) { - std::string file_path = - cc->InputSidePackets().Get(input_id).Get(); - ASSIGN_OR_RETURN(file_path, PathToResourceAsFile(file_path)); - - std::string contents; - MP_RETURN_IF_ERROR(GetResourceContents( - file_path, &contents, /*read_as_binary=*/!options.text_mode())); - cc->OutputSidePackets().Get(output_id).Set( - MakePacket(std::move(contents))); - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; - -REGISTER_CALCULATOR(LocalFileContentsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/local_file_contents_calculator.proto b/mediapipe/calculators/util/local_file_contents_calculator.proto deleted file mode 100644 index 17876c89f..000000000 --- a/mediapipe/calculators/util/local_file_contents_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LocalFileContentsCalculatorOptions { - extend CalculatorOptions { - optional LocalFileContentsCalculatorOptions ext = 346849340; - } - - // By default, set the file open mode to 'rb'. Otherwise, set the mode to 'r'. - optional bool text_mode = 1; -} diff --git a/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc b/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc deleted file mode 100644 index fcba83a49..000000000 --- a/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -// The calculator takes the path to local directory and desired file suffix to -// mach as input side packets, and outputs the contents of those files that -// match the pattern. Those matched files will be sent sequentially through the -// output stream with incremental timestamp difference by 1. -// -// Example config: -// node { -// calculator: "LocalFilePatternContentsCalculator" -// input_side_packet: "FILE_DIRECTORY:file_directory" -// input_side_packet: "FILE_SUFFIX:file_suffix" -// output_stream: "CONTENTS:contents" -// } -class LocalFilePatternContentsCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag("FILE_DIRECTORY").Set(); - cc->InputSidePackets().Tag("FILE_SUFFIX").Set(); - cc->Outputs().Tag("CONTENTS").Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - MP_RETURN_IF_ERROR(mediapipe::file::MatchFileTypeInDirectory( - cc->InputSidePackets().Tag("FILE_DIRECTORY").Get(), - cc->InputSidePackets().Tag("FILE_SUFFIX").Get(), - &filenames_)); - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - if (current_output_ < filenames_.size()) { - auto contents = absl::make_unique(); - LOG(INFO) << filenames_[current_output_]; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - filenames_[current_output_], contents.get())); - ++current_output_; - cc->Outputs() - .Tag("CONTENTS") - .Add(contents.release(), Timestamp(current_output_)); - } else { - return tool::StatusStop(); - } - return absl::OkStatus(); - } - - private: - std::vector filenames_; - int current_output_ = 0; -}; - -REGISTER_CALCULATOR(LocalFilePatternContentsCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/logic_calculator.cc b/mediapipe/calculators/util/logic_calculator.cc deleted file mode 100644 index d9bb9281a..000000000 --- a/mediapipe/calculators/util/logic_calculator.cc +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/calculators/util/logic_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -using mediapipe::LogicCalculatorOptions; - -// A calculator to compute logical functions of bool inputs. -// With just one input, the output equals the input as expected. -// -// Inputs: One or more bool inputs, which may be input-stream-packets, -// input-side-packets, or options input-values. -// -// Outputs: One bool stream. -// -// Example config: -// node { -// calculator: "LogicCalculator" -// input_stream: "has_data" -// input_side_packet: "enable" -// input_stream: "is_valid" -// output_stream: "process_data" -// options { -// [mediapipe.LogicCalculatorOptions.ext] { -// op: AND -// input_value: true -// } -// } -// } -class LogicCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - for (int k = 0; k < cc->Inputs().NumEntries(""); ++k) { - cc->Inputs().Index(k).Set(); - } - for (int k = 0; k < cc->InputSidePackets().NumEntries(""); ++k) { - cc->InputSidePackets().Index(k).Set(); - } - RET_CHECK_GE(cc->Inputs().NumEntries("") + - cc->InputSidePackets().NumEntries("") + - cc->Options().input_value_size(), - 1); - RET_CHECK_EQ(cc->Outputs().NumEntries(""), 1); - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - options_ = cc->Options(); - cc->SetOffset(TimestampDiff(0)); - return absl::OkStatus(); - } - - bool LogicalOp(bool b1, bool b2) { - switch (options_.op()) { - case LogicCalculatorOptions::AND: - return b1 && b2; - case LogicCalculatorOptions::OR: - return b1 || b2; - case LogicCalculatorOptions::XOR: - return b1 ^ b2; - } - return false; - } - - absl::Status Process(CalculatorContext* cc) override { - bool result = options_.op() == LogicCalculatorOptions::AND ? true : false; - for (int k = 0; k < options_.input_value_size(); ++k) { - result = LogicalOp(result, options_.input_value(k)); - } - for (int k = 0; k < cc->Inputs().NumEntries(""); ++k) { - result = LogicalOp(result, cc->Inputs().Index(k).Value().Get()); - } - for (int k = 0; k < cc->InputSidePackets().NumEntries(""); ++k) { - result = LogicalOp(result, cc->InputSidePackets().Index(k).Get()); - } - if (options_.negate()) { - result = !result; - } - cc->Outputs().Index(0).Add(new bool(result), cc->InputTimestamp()); - return absl::OkStatus(); - } - - private: - LogicCalculatorOptions options_; -}; -REGISTER_CALCULATOR(LogicCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/logic_calculator.proto b/mediapipe/calculators/util/logic_calculator.proto deleted file mode 100644 index fe00a2d9b..000000000 --- a/mediapipe/calculators/util/logic_calculator.proto +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message LogicCalculatorOptions { - extend CalculatorOptions { - optional LogicCalculatorOptions ext = 338731246; - } - // The logical operation to apply. - enum Operation { - AND = 0; - OR = 1; - XOR = 2; - } - optional Operation op = 1; - - // Whether to negate the result. - optional bool negate = 2; - - // Optional bool input values. - repeated bool input_value = 3; -} diff --git a/mediapipe/calculators/util/non_max_suppression_calculator.cc b/mediapipe/calculators/util/non_max_suppression_calculator.cc deleted file mode 100644 index 535e2a719..000000000 --- a/mediapipe/calculators/util/non_max_suppression_calculator.cc +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "mediapipe/calculators/util/non_max_suppression_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/rectangle.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -typedef std::vector Detections; -typedef std::vector> IndexedScores; - -namespace { - -constexpr char kImageTag[] = "IMAGE"; - -bool SortBySecond(const std::pair& indexed_score_0, - const std::pair& indexed_score_1) { - return (indexed_score_0.second > indexed_score_1.second); -} - -// Removes all but the max scoring label and its score from the detection. -// Returns true if the detection has at least one label. -bool RetainMaxScoringLabelOnly(Detection* detection) { - if (detection->label_id_size() == 0 && detection->label_size() == 0) { - return false; - } - CHECK(detection->label_id_size() == detection->score_size() || - detection->label_size() == detection->score_size()) - << "Number of scores must be equal to number of detections."; - - std::vector> indexed_scores; - indexed_scores.reserve(detection->score_size()); - for (int k = 0; k < detection->score_size(); ++k) { - indexed_scores.push_back(std::make_pair(k, detection->score(k))); - } - std::sort(indexed_scores.begin(), indexed_scores.end(), SortBySecond); - const int top_index = indexed_scores[0].first; - detection->clear_score(); - detection->add_score(indexed_scores[0].second); - if (detection->label_id_size() > top_index) { - const int top_label_id = detection->label_id(top_index); - detection->clear_label_id(); - detection->add_label_id(top_label_id); - } else { - const std::string top_label = detection->label(top_index); - detection->clear_label(); - detection->add_label(top_label); - } - - return true; -} - -// Computes an overlap similarity between two rectangles. Similarity measure is -// defined by overlap_type parameter. -float OverlapSimilarity( - const NonMaxSuppressionCalculatorOptions::OverlapType overlap_type, - const Rectangle_f& rect1, const Rectangle_f& rect2) { - if (!rect1.Intersects(rect2)) return 0.0f; - const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area(); - float normalization; - switch (overlap_type) { - case NonMaxSuppressionCalculatorOptions::JACCARD: - normalization = Rectangle_f(rect1).Union(rect2).Area(); - break; - case NonMaxSuppressionCalculatorOptions::MODIFIED_JACCARD: - normalization = rect2.Area(); - break; - case NonMaxSuppressionCalculatorOptions::INTERSECTION_OVER_UNION: - normalization = rect1.Area() + rect2.Area() - intersection_area; - break; - default: - LOG(FATAL) << "Unrecognized overlap type: " << overlap_type; - } - return normalization > 0.0f ? intersection_area / normalization : 0.0f; -} - -// Computes an overlap similarity between two locations by first extracting the -// relative box (dimension normalized by frame width/height) from the location. -float OverlapSimilarity( - const int frame_width, const int frame_height, - const NonMaxSuppressionCalculatorOptions::OverlapType overlap_type, - const Location& location1, const Location& location2) { - const auto rect1 = location1.ConvertToRelativeBBox(frame_width, frame_height); - const auto rect2 = location2.ConvertToRelativeBBox(frame_width, frame_height); - return OverlapSimilarity(overlap_type, rect1, rect2); -} - -// Computes an overlap similarity between two locations by first extracting the -// relative box from the location. It assumes that a relative-box representation -// is already available in the location, and therefore frame width and height -// are not needed for further normalization. -float OverlapSimilarity( - const NonMaxSuppressionCalculatorOptions::OverlapType overlap_type, - const Location& location1, const Location& location2) { - const auto rect1 = location1.GetRelativeBBox(); - const auto rect2 = location2.GetRelativeBBox(); - return OverlapSimilarity(overlap_type, rect1, rect2); -} - -} // namespace - -// A calculator performing non-maximum suppression on a set of detections. -// Inputs: -// 1. IMAGE (optional): A stream of ImageFrame used to obtain the frame size. -// No image data is used. Not needed if the detection bounding boxes are -// already represented in normalized dimensions (0.0~1.0). -// 2. A variable number of input streams of type std::vector. The -// exact number of such streams should be set via num_detection_streams -// field in the calculator options. -// -// Outputs: a single stream of type std::vector containing a subset -// of the input detections after non-maximum suppression. -// -// Example config: -// node { -// calculator: "NonMaxSuppressionCalculator" -// input_stream: "IMAGE:frames" -// input_stream: "detections1" -// input_stream: "detections2" -// output_stream: "detections" -// options { -// [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { -// num_detection_streams: 2 -// max_num_detections: 10 -// min_suppression_threshold: 0.2 -// overlap_type: JACCARD -// } -// } -// } -class NonMaxSuppressionCalculator : public CalculatorBase { - public: - NonMaxSuppressionCalculator() = default; - ~NonMaxSuppressionCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - if (cc->Inputs().HasTag(kImageTag)) { - cc->Inputs().Tag(kImageTag).Set(); - } - for (int k = 0; k < options.num_detection_streams(); ++k) { - cc->Inputs().Index(k).Set(); - } - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - CHECK_GT(options_.num_detection_streams(), 0) - << "At least one detection stream need to be specified."; - CHECK_NE(options_.max_num_detections(), 0) - << "max_num_detections=0 is not a valid value. Please choose a " - << "positive number of you want to limit the number of output " - << "detections, or set -1 if you do not want any limit."; - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - // Add all input detections to the same vector. - Detections input_detections; - for (int i = 0; i < options_.num_detection_streams(); ++i) { - const auto& detections_packet = cc->Inputs().Index(i).Value(); - // Check whether this stream has a packet for this timestamp. - if (detections_packet.IsEmpty()) { - continue; - } - const auto& detections = detections_packet.Get(); - - input_detections.insert(input_detections.end(), detections.begin(), - detections.end()); - } - - // Check if there are any detections at all. - if (input_detections.empty()) { - if (options_.return_empty_detections()) { - cc->Outputs().Index(0).Add(new Detections(), cc->InputTimestamp()); - } - return absl::OkStatus(); - } - - // Remove all but the maximum scoring label from each input detection. This - // corresponds to non-maximum suppression among detections which have - // identical locations. - Detections pruned_detections; - pruned_detections.reserve(input_detections.size()); - for (auto& detection : input_detections) { - if (RetainMaxScoringLabelOnly(&detection)) { - pruned_detections.push_back(detection); - } - } - - // Copy all the scores (there is a single score in each detection after - // the above pruning) to an indexed vector for sorting. The first value is - // the index of the detection in the original vector from which the score - // stems, while the second is the actual score. - IndexedScores indexed_scores; - indexed_scores.reserve(pruned_detections.size()); - for (int index = 0; index < pruned_detections.size(); ++index) { - indexed_scores.push_back( - std::make_pair(index, pruned_detections[index].score(0))); - } - std::sort(indexed_scores.begin(), indexed_scores.end(), SortBySecond); - - const int max_num_detections = - (options_.max_num_detections() > -1) - ? options_.max_num_detections() - : static_cast(indexed_scores.size()); - // A set of detections and locations, wrapping the location data from each - // detection, which are retained after the non-maximum suppression. - auto* retained_detections = new Detections(); - retained_detections->reserve(max_num_detections); - - if (options_.algorithm() == NonMaxSuppressionCalculatorOptions::WEIGHTED) { - WeightedNonMaxSuppression(indexed_scores, pruned_detections, - max_num_detections, cc, retained_detections); - } else { - NonMaxSuppression(indexed_scores, pruned_detections, max_num_detections, - cc, retained_detections); - } - - cc->Outputs().Index(0).Add(retained_detections, cc->InputTimestamp()); - - return absl::OkStatus(); - } - - private: - void NonMaxSuppression(const IndexedScores& indexed_scores, - const Detections& detections, int max_num_detections, - CalculatorContext* cc, Detections* output_detections) { - std::vector retained_locations; - retained_locations.reserve(max_num_detections); - // We traverse the detections by decreasing score. - for (const auto& indexed_score : indexed_scores) { - const auto& detection = detections[indexed_score.first]; - if (options_.min_score_threshold() > 0 && - detection.score(0) < options_.min_score_threshold()) { - break; - } - const Location location(detection.location_data()); - bool suppressed = false; - // The current detection is suppressed iff there exists a retained - // detection, whose location overlaps more than the specified - // threshold with the location of the current detection. - for (const auto& retained_location : retained_locations) { - float similarity; - if (cc->Inputs().HasTag(kImageTag)) { - const auto& frame = cc->Inputs().Tag(kImageTag).Get(); - similarity = OverlapSimilarity(frame.Width(), frame.Height(), - options_.overlap_type(), - retained_location, location); - } else { - similarity = OverlapSimilarity(options_.overlap_type(), - retained_location, location); - } - if (similarity > options_.min_suppression_threshold()) { - suppressed = true; - break; - } - } - if (!suppressed) { - output_detections->push_back(detection); - retained_locations.push_back(location); - } - if (output_detections->size() >= max_num_detections) { - break; - } - } - } - - void WeightedNonMaxSuppression(const IndexedScores& indexed_scores, - const Detections& detections, - int max_num_detections, CalculatorContext* cc, - Detections* output_detections) { - IndexedScores remained_indexed_scores; - remained_indexed_scores.assign(indexed_scores.begin(), - indexed_scores.end()); - - IndexedScores remained; - IndexedScores candidates; - output_detections->clear(); - while (!remained_indexed_scores.empty()) { - const int original_indexed_scores_size = remained_indexed_scores.size(); - const auto& detection = detections[remained_indexed_scores[0].first]; - if (options_.min_score_threshold() > 0 && - detection.score(0) < options_.min_score_threshold()) { - break; - } - remained.clear(); - candidates.clear(); - const Location location(detection.location_data()); - // This includes the first box. - for (const auto& indexed_score : remained_indexed_scores) { - Location rest_location(detections[indexed_score.first].location_data()); - float similarity = - OverlapSimilarity(options_.overlap_type(), rest_location, location); - if (similarity > options_.min_suppression_threshold()) { - candidates.push_back(indexed_score); - } else { - remained.push_back(indexed_score); - } - } - auto weighted_detection = detection; - if (!candidates.empty()) { - const int num_keypoints = - detection.location_data().relative_keypoints_size(); - std::vector keypoints(num_keypoints * 2); - float w_xmin = 0.0f; - float w_ymin = 0.0f; - float w_xmax = 0.0f; - float w_ymax = 0.0f; - float total_score = 0.0f; - for (const auto& candidate : candidates) { - total_score += candidate.second; - const auto& location_data = - detections[candidate.first].location_data(); - const auto& bbox = location_data.relative_bounding_box(); - w_xmin += bbox.xmin() * candidate.second; - w_ymin += bbox.ymin() * candidate.second; - w_xmax += (bbox.xmin() + bbox.width()) * candidate.second; - w_ymax += (bbox.ymin() + bbox.height()) * candidate.second; - - for (int i = 0; i < num_keypoints; ++i) { - keypoints[i * 2] += - location_data.relative_keypoints(i).x() * candidate.second; - keypoints[i * 2 + 1] += - location_data.relative_keypoints(i).y() * candidate.second; - } - } - auto* weighted_location = weighted_detection.mutable_location_data() - ->mutable_relative_bounding_box(); - weighted_location->set_xmin(w_xmin / total_score); - weighted_location->set_ymin(w_ymin / total_score); - weighted_location->set_width((w_xmax / total_score) - - weighted_location->xmin()); - weighted_location->set_height((w_ymax / total_score) - - weighted_location->ymin()); - for (int i = 0; i < num_keypoints; ++i) { - auto* keypoint = weighted_detection.mutable_location_data() - ->mutable_relative_keypoints(i); - keypoint->set_x(keypoints[i * 2] / total_score); - keypoint->set_y(keypoints[i * 2 + 1] / total_score); - } - } - - output_detections->push_back(weighted_detection); - // Breaks the loop if the size of indexed scores doesn't change after an - // iteration. - if (original_indexed_scores_size == remained.size()) { - break; - } else { - remained_indexed_scores = std::move(remained); - } - } - } - - NonMaxSuppressionCalculatorOptions options_; -}; -REGISTER_CALCULATOR(NonMaxSuppressionCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/non_max_suppression_calculator.proto b/mediapipe/calculators/util/non_max_suppression_calculator.proto deleted file mode 100644 index 5fa960497..000000000 --- a/mediapipe/calculators/util/non_max_suppression_calculator.proto +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Options to NonMaxSuppression calculator, which performs non-maximum -// suppression on a set of detections. -message NonMaxSuppressionCalculatorOptions { - extend CalculatorOptions { - optional NonMaxSuppressionCalculatorOptions ext = 55383100; - } - - // Number of input streams. Each input stream should contain a vector of - // detections. - optional int32 num_detection_streams = 1 [default = 1]; - - // Maximum number of detections to be returned. If -1, then all detections are - // returned. - optional int32 max_num_detections = 2 [default = -1]; - - // Minimum score of detections to be returned. - optional float min_score_threshold = 6 [default = -1.0]; - - // Jaccard similarity threshold for suppression -- a detection would suppress - // all other detections whose scores are lower and overlap by at least the - // specified threshold. - optional float min_suppression_threshold = 3 [default = 1.0]; - - // During the overlap computation, which is used to determine whether a - // rectangle suppresses another rectangle, one can use the Jaccard similarity, - // defined as the ration of the intersection over union of the two rectangles. - // Alternatively a modified version of Jaccard can be used, where the - // normalization is done by the area of the rectangle being checked for - // suppression. - enum OverlapType { - UNSPECIFIED_OVERLAP_TYPE = 0; - JACCARD = 1; - MODIFIED_JACCARD = 2; - INTERSECTION_OVER_UNION = 3; - } - optional OverlapType overlap_type = 4 [default = JACCARD]; - - // Whether to put empty detection vector in output stream. - optional bool return_empty_detections = 5; - - // Algorithms that can be used to apply non-maximum suppression. - enum NmsAlgorithm { - DEFAULT = 0; - // Only supports relative bounding box for weighted NMS. - WEIGHTED = 1; - } - optional NmsAlgorithm algorithm = 7 [default = DEFAULT]; -} diff --git a/mediapipe/calculators/util/packet_frequency.proto b/mediapipe/calculators/util/packet_frequency.proto deleted file mode 100644 index 177a73b12..000000000 --- a/mediapipe/calculators/util/packet_frequency.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto2"; - -package mediapipe; - -// Contains the packet frequency information. -message PacketFrequency { - // Packet frequency (packets per second). - optional double packet_frequency_hz = 1; - - // A label that identifies what this packet frequency is for. Eg. "Gaze", - // "Gesture", etc. - optional string label = 2; -} diff --git a/mediapipe/calculators/util/packet_frequency_calculator.cc b/mediapipe/calculators/util/packet_frequency_calculator.cc deleted file mode 100644 index 19ffae70e..000000000 --- a/mediapipe/calculators/util/packet_frequency_calculator.cc +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/time/time.h" -#include "mediapipe/calculators/util/packet_frequency.pb.h" -#include "mediapipe/calculators/util/packet_frequency_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace { -constexpr int kSecondsToMicroseconds = 1000000; - -} // namespace - -namespace mediapipe { -// A MediaPipe calculator that computes the frequency (in Hertz) of incoming -// packet streams. The frequency of packets is computed over a time window -// that is configured in options. There must be one output stream corresponding -// to every input packet stream. The frequency is output as a PacketFrequency -// proto. -// -// NOTE: -// 1. For computing frequency, packet timestamps are used and not the wall -// timestamp. Hence, the calculator is best-suited for real-time applications. -// 2. When multiple input/output streams are present, the calculator must be -// used with an ImmediateInputStreamHandler. -// -// Example config: -// node { -// calculator: "PacketFrequencyCalculator" -// input_stream: "input_stream_0" -// input_stream: "input_stream_1" -// . -// . -// input_stream: "input_stream_N" -// output_stream: "packet_frequency_0" -// output_stream: "packet_frequency_1" -// . -// . -// output_stream: "packet_frequency_N" -// input_stream_handler { -// input_stream_handler: "ImmediateInputStreamHandler" -// } -// options { -// [soapbox.PacketFrequencyCalculatorOptions.ext] { -// time_window_sec: 3.0 -// label: "stream_name_0" -// label: "stream_name_1" -// . -// . -// label: "stream_name_N" -// } -// } -// } -class PacketFrequencyCalculator : public CalculatorBase { - public: - PacketFrequencyCalculator() {} - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Outputs the given framerate on the specified output stream as a - // PacketFrequency proto. - absl::Status OutputPacketFrequency(CalculatorContext* cc, int stream_id, - double framerate_hz, - const std::string& label, - const Timestamp& input_timestamp); - - // Adds the input timestamp in the particular stream's timestamp buffer. - absl::Status AddPacketTimestampForStream(int stream_id, int64 timestamp); - - // For the specified input stream, clears timestamps from buffer that are - // older than the configured time_window_sec. - absl::Status ClearOldpacketTimestamps(int stream_id, int64 current_timestamp); - - // Options for the calculator. - PacketFrequencyCalculatorOptions options_; - - // Map where key is the input stream ID and value is the timestamp of the - // first packet received on that stream. - std::map first_timestamp_for_stream_id_usec_; - - // Map where key is the input stream ID and value is a vector that stores - // timestamps of recently received packets on the stream. Timestamps older - // than the time_window_sec are continuously deleted for all the streams. - std::map> previous_timestamps_for_stream_id_; -}; -REGISTER_CALCULATOR(PacketFrequencyCalculator); - -absl::Status PacketFrequencyCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ(cc->Outputs().NumEntries(), cc->Inputs().NumEntries()); - for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - cc->Inputs().Index(i).SetAny(); - cc->Outputs().Index(i).Set(); - } - return absl::OkStatus(); -} - -absl::Status PacketFrequencyCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - RET_CHECK_EQ(options_.label_size(), cc->Inputs().NumEntries()); - RET_CHECK_GT(options_.time_window_sec(), 0); - RET_CHECK_LE(options_.time_window_sec(), 100); - - // Initialize the stream-related data structures. - for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - RET_CHECK(!options_.label(i).empty()); - previous_timestamps_for_stream_id_[i] = {}; - first_timestamp_for_stream_id_usec_[i] = -1; - } - return absl::OkStatus(); -} - -absl::Status PacketFrequencyCalculator::Process(CalculatorContext* cc) { - for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - if (cc->Inputs().Index(i).IsEmpty()) { - continue; - } - RET_CHECK_OK(AddPacketTimestampForStream(/*stream_id=*/i, - cc->InputTimestamp().Value())); - RET_CHECK_OK(ClearOldpacketTimestamps(/*stream_id=*/i, - cc->InputTimestamp().Value())); - - if (first_timestamp_for_stream_id_usec_[i] < 0) { - first_timestamp_for_stream_id_usec_[i] = cc->InputTimestamp().Value(); - - // Since this is the very first packet on this stream, we don't have a - // window of time over which we can compute the packet frequency. So - // outputting packet frequency for this stream as 0 Hz. - return OutputPacketFrequency(cc, /*stream_id=*/i, /*framerate_hz=*/0.0, - options_.label(i), cc->InputTimestamp()); - } - - // If the time elapsed is less that the configured time window, then use - // that time duration instead, else use the configured time window. - double time_window_usec = - std::min(static_cast(cc->InputTimestamp().Value() - - first_timestamp_for_stream_id_usec_[i]), - options_.time_window_sec() * kSecondsToMicroseconds); - - double framerate_hz = (previous_timestamps_for_stream_id_[i].size() * 1.0) / - (time_window_usec / kSecondsToMicroseconds); - - return OutputPacketFrequency(cc, /*stream_id=*/i, framerate_hz, - options_.label(i), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status PacketFrequencyCalculator::AddPacketTimestampForStream( - int stream_id, int64 timestamp_usec) { - if (previous_timestamps_for_stream_id_.find(stream_id) == - previous_timestamps_for_stream_id_.end()) { - return absl::InvalidArgumentError("Input stream id is invalid"); - } - - previous_timestamps_for_stream_id_[stream_id].push_back(timestamp_usec); - - return absl::OkStatus(); -} - -absl::Status PacketFrequencyCalculator::ClearOldpacketTimestamps( - int stream_id, int64 current_timestamp_usec) { - if (previous_timestamps_for_stream_id_.find(stream_id) == - previous_timestamps_for_stream_id_.end()) { - return absl::InvalidArgumentError("Input stream id is invalid"); - } - - auto& timestamps_buffer = previous_timestamps_for_stream_id_[stream_id]; - int64 time_window_usec = options_.time_window_sec() * kSecondsToMicroseconds; - - timestamps_buffer.erase( - std::remove_if(timestamps_buffer.begin(), timestamps_buffer.end(), - [&time_window_usec, - ¤t_timestamp_usec](const int64 timestamp_usec) { - return current_timestamp_usec - timestamp_usec > - time_window_usec; - }), - timestamps_buffer.end()); - - return absl::OkStatus(); -} - -absl::Status PacketFrequencyCalculator::OutputPacketFrequency( - CalculatorContext* cc, int stream_id, double framerate_hz, - const std::string& label, const Timestamp& input_timestamp) { - auto packet_frequency = absl::make_unique(); - packet_frequency->set_packet_frequency_hz(framerate_hz); - packet_frequency->set_label(label); - - cc->Outputs().Index(stream_id).Add(packet_frequency.release(), - input_timestamp); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/packet_frequency_calculator.proto b/mediapipe/calculators/util/packet_frequency_calculator.proto deleted file mode 100644 index e7be1c420..000000000 --- a/mediapipe/calculators/util/packet_frequency_calculator.proto +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Options for PacketFrequencyCalculator. -message PacketFrequencyCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional PacketFrequencyCalculatorOptions ext = 168468918; - } - - // Time window (in seconds) over which the packet frequency is computed. Must - // be greater than 0 and less than 100 seconds (in order to limit memory - // usage). - optional double time_window_sec = 1 [default = 3.0]; - - // Text identifiers for the input streams. - repeated string label = 2; -} diff --git a/mediapipe/calculators/util/packet_frequency_calculator_test.cc b/mediapipe/calculators/util/packet_frequency_calculator_test.cc deleted file mode 100644 index f8e7c0236..000000000 --- a/mediapipe/calculators/util/packet_frequency_calculator_test.cc +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/packet_frequency.pb.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { -namespace { - -CalculatorGraphConfig::Node GetDefaultNode() { - return ParseTextProtoOrDie(R"pb( - calculator: "PacketFrequencyCalculator" - input_stream: "packet_stream" - output_stream: "packet_frequency" - options { - [mediapipe.PacketFrequencyCalculatorOptions.ext] { - time_window_sec: 3.0 - label: "stream_description" - } - } - )pb"); -} - -CalculatorGraphConfig::Node GetNodeWithMultipleStreams() { - return ParseTextProtoOrDie(R"pb( - calculator: "PacketFrequencyCalculator" - input_stream: "packet_stream_0" - input_stream: "packet_stream_1" - input_stream: "packet_stream_2" - output_stream: "packet_frequency_0" - output_stream: "packet_frequency_1" - output_stream: "packet_frequency_2" - input_stream_handler { input_stream_handler: "ImmediateInputStreamHandler" } - options { - [mediapipe.PacketFrequencyCalculatorOptions.ext] { - time_window_sec: 3.0 - label: "stream_description_0" - label: "stream_description_1" - label: "stream_description_2" - } - } - )pb"); -} - -// Tests packet frequency. -TEST(PacketFrequencyCalculatorTest, MultiPacketTest) { - // Setup the calculator runner and provide integer packets as input (note that - // it doesn't have to be integer; the calculator can take any type as input). - CalculatorRunner runner(GetDefaultNode()); - - // Packet 1. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(0))); - // Packet 2. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(500000))); - // Packet 3. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(1000000))); - // Packet 4. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(1500000))); - // Packet 5. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(3000000))); - // Packet 6. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(4000000))); - // Packet 7. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new int).At(Timestamp(9000000))); - - // Run the calculator. - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output_packets = runner.Outputs().Index(0).packets; - - // Very first packet. So frequency is zero. - const auto& output1 = output_packets[0].Get(); - EXPECT_FLOAT_EQ(output1.packet_frequency_hz(), 0.0); - EXPECT_EQ(output1.label(), "stream_description"); - - // 2 packets in the first 500ms. - const auto& output2 = output_packets[1].Get(); - EXPECT_FLOAT_EQ(output2.packet_frequency_hz(), 4.000000); - EXPECT_EQ(output2.label(), "stream_description"); - - // 3 packets in the first 1 sec. - const auto& output3 = output_packets[2].Get(); - EXPECT_FLOAT_EQ(output3.packet_frequency_hz(), 3.000000); - EXPECT_EQ(output3.label(), "stream_description"); - - // 4 packets in the first 1.5 sec. - const auto& output4 = output_packets[3].Get(); - EXPECT_FLOAT_EQ(output4.packet_frequency_hz(), 2.666667); - EXPECT_EQ(output4.label(), "stream_description"); - - // 5 packets in the first 3 sec. - const auto& output5 = output_packets[4].Get(); - EXPECT_FLOAT_EQ(output5.packet_frequency_hz(), 1.666667); - EXPECT_EQ(output5.label(), "stream_description"); - - // 4 packets in the past 3 sec window. - const auto& output6 = output_packets[5].Get(); - EXPECT_FLOAT_EQ(output6.packet_frequency_hz(), 1.333333); - EXPECT_EQ(output6.label(), "stream_description"); - - // 1 packet in the past 3 sec window. - const auto& output7 = output_packets[6].Get(); - EXPECT_FLOAT_EQ(output7.packet_frequency_hz(), 0.33333334); - EXPECT_EQ(output7.label(), "stream_description"); -} - -// Tests packet frequency with multiple input/output streams. -TEST(PacketFrequencyCalculatorTest, MultiStreamTest) { - // Setup the calculator runner and provide strings as input on all streams - // (note that it doesn't have to be std::string; the calculator can take any - // type as input). - CalculatorRunner runner(GetNodeWithMultipleStreams()); - - // Packet 1 on stream 1. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string).At(Timestamp(250000))); - // Packet 2 on stream 1. - runner.MutableInputs()->Index(0).packets.push_back( - Adopt(new std::string).At(Timestamp(500000))); - // Packet 1 on stream 2. - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(new std::string).At(Timestamp(100000))); - // Packet 2 on stream 2. - runner.MutableInputs()->Index(1).packets.push_back( - Adopt(new std::string).At(Timestamp(5000000))); - // Packet 1 on stream 3. - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(new std::string).At(Timestamp(0))); - // Packet 2 on stream 3. - runner.MutableInputs()->Index(2).packets.push_back( - Adopt(new std::string).At(Timestamp(3000000))); - - // Run the calculator. - MP_ASSERT_OK(runner.Run()) << "Calculator execution failed."; - const std::vector& output_packets_stream_1 = - runner.Outputs().Index(0).packets; - const std::vector& output_packets_stream_2 = - runner.Outputs().Index(1).packets; - const std::vector& output_packets_stream_3 = - runner.Outputs().Index(2).packets; - - // First packet on stream 1. So frequency is zero. - const auto& output1 = output_packets_stream_1[0].Get(); - EXPECT_FLOAT_EQ(output1.packet_frequency_hz(), 0.0); - EXPECT_EQ(output1.label(), "stream_description_0"); - - // Second packet on stream 1. - const auto& output2 = output_packets_stream_1[1].Get(); - EXPECT_FLOAT_EQ(output2.packet_frequency_hz(), 8.000000); - EXPECT_EQ(output2.label(), "stream_description_0"); - - // First packet on stream 2. So frequency is zero. - const auto& output3 = output_packets_stream_2[0].Get(); - EXPECT_FLOAT_EQ(output3.packet_frequency_hz(), 0.0); - EXPECT_EQ(output3.label(), "stream_description_1"); - - // Second packet on stream 2. - const auto& output4 = output_packets_stream_2[1].Get(); - EXPECT_FLOAT_EQ(output4.packet_frequency_hz(), 0.33333334); - EXPECT_EQ(output4.label(), "stream_description_1"); - - // First packet on stream 3. So frequency is zero. - const auto& output5 = output_packets_stream_3[0].Get(); - EXPECT_FLOAT_EQ(output5.packet_frequency_hz(), 0.0); - EXPECT_EQ(output5.label(), "stream_description_2"); - - // Second packet on stream 3. - const auto& output6 = output_packets_stream_3[1].Get(); - EXPECT_FLOAT_EQ(output6.packet_frequency_hz(), 0.66666669); - EXPECT_EQ(output6.label(), "stream_description_2"); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/packet_latency_calculator.cc b/mediapipe/calculators/util/packet_latency_calculator.cc deleted file mode 100644 index 35e415505..000000000 --- a/mediapipe/calculators/util/packet_latency_calculator.cc +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/str_cat.h" -#include "absl/time/time.h" -#include "mediapipe/calculators/util/latency.pb.h" -#include "mediapipe/calculators/util/packet_latency_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { - -namespace { - -// Tag name for clock side packet. -constexpr char kClockTag[] = "CLOCK"; - -// Tag name for reference signal. -constexpr char kReferenceSignalTag[] = "REFERENCE_SIGNAL"; -} // namespace - -// A MediaPipe calculator that computes latency of incoming packet streams with -// respect to a reference signal (e.g image, audio frames). -// -// The latency of a packet wrt a reference packet is defined as the difference -// between arrival times of the two. A latency of X microseconds implies that -// the packet arrived X microseconds after its corresponding reference packet. -// For each packet stream, the calculator outputs the current latency, average, -// and a histogram of observed latencies so far. -// -// NOTE: -// 1) This calculator is meant to be used ONLY with an -// ImmediateInputStreamHandler. -// 2) This calculator is meant to be used only for real-time or simulated real- -// time applications. For example, the reference signal could be audio/video -// frames coming from a calculator that reads microphone/webcam data or some -// calculator that simulates real-time input. -// 3) If the packet labels are provided through options, then the number of -// labels should be exactly same as number of output_streams. If no packet -// label is defined in the node options, the calculator uses the input stream -// names. -// -// InputSidePacket (Optional): -// CLOCK: A clock for knowing current time. -// -// Inputs: -// 0- Packet stream 0 (e.g image feature 0): -// 1- Packet stream 1 (e.g image features 1): -// ... -// N- Packet stream N (e.g image features N): -// REFERENCE_SIGNAL: The reference signal from which the above packets were -// extracted (e.g image frames). -// -// Outputs: -// 0- Latency of packet stream 0. -// 1- Latency of packet stream 1. -// ... -// N- Latency of packet stream N. -// -// Example config: -// node { -// calculator: "PacketLatencyCalculator" -// input_side_packet: "monotonic_clock" -// input_stream: "packet_stream_0" -// input_stream: "packet_stream_1" -// ... -// input_stream: "packet_stream_N" -// input_stream: "REFERENCE_SIGNAL:camera_frames" -// output_stream: "packet_latency_0" -// output_stream: "packet_latency_1" -// ... -// output_stream: "packet_latency_N" -// options { -// [soapbox.PacketLatencyCalculatorOptions.ext] { -// num_intervals: 10 -// interval_size_usec: 10000 -// } -// } -// input_stream_handler { -// input_stream_handler: 'ImmediateInputStreamHandler' -// } -// } -class PacketLatencyCalculator : public CalculatorBase { - public: - PacketLatencyCalculator() {} - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - // Resets the histogram and running average variables by initializing them to - // zero. - void ResetStatistics(); - - // Calculator options. - PacketLatencyCalculatorOptions options_; - - // Clock object. - std::shared_ptr<::mediapipe::Clock> clock_; - - // Clock time when the first reference packet was received. - int64 first_process_time_usec_ = -1; - - // Timestamp of the first reference packet received. - int64 first_reference_timestamp_usec_ = -1; - - // Number of packet streams. - int64 num_packet_streams_ = -1; - - // Latency output for each packet stream. - std::vector packet_latencies_; - - // Running sum and count of latencies for each packet stream. This is required - // to compute the average latency. - std::vector sum_latencies_usec_; - std::vector num_latencies_; - - // Clock time when last reset was done for histogram and running average. - int64 last_reset_time_usec_ = -1; -}; -REGISTER_CALCULATOR(PacketLatencyCalculator); - -absl::Status PacketLatencyCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_GT(cc->Inputs().NumEntries(), 1); - - // Input and output streams. - int64 num_packet_streams = cc->Inputs().NumEntries() - 1; - RET_CHECK_EQ(cc->Outputs().NumEntries(), num_packet_streams); - for (int64 i = 0; i < num_packet_streams; ++i) { - cc->Inputs().Index(i).SetAny(); - cc->Outputs().Index(i).Set(); - } - - // Reference signal. - cc->Inputs().Tag(kReferenceSignalTag).SetAny(); - - // Clock side packet. - if (cc->InputSidePackets().HasTag(kClockTag)) { - cc->InputSidePackets() - .Tag(kClockTag) - .Set>(); - } - - return absl::OkStatus(); -} - -void PacketLatencyCalculator::ResetStatistics() { - // Initialize histogram with zero counts and set running average to zero. - for (int64 i = 0; i < num_packet_streams_; ++i) { - for (int64 interval_index = 0; interval_index < options_.num_intervals(); - ++interval_index) { - packet_latencies_[i].set_counts(interval_index, 0); - } - - // Initialize the running sum and count to 0. - sum_latencies_usec_[i] = 0; - num_latencies_[i] = 0; - } -} - -absl::Status PacketLatencyCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - num_packet_streams_ = cc->Inputs().NumEntries() - 1; - - // Check if provided labels are of correct size. - bool labels_provided = !options_.packet_labels().empty(); - if (labels_provided) { - RET_CHECK_EQ(options_.packet_labels_size(), num_packet_streams_) - << "Input packet stream count different from output stream count."; - } - - // Check that histogram params are valid. - RET_CHECK_GT(options_.num_intervals(), 0); - RET_CHECK_GT(options_.interval_size_usec(), 0); - - // Initialize latency outputs for all streams. - packet_latencies_.resize(num_packet_streams_); - sum_latencies_usec_.resize(num_packet_streams_); - num_latencies_.resize(num_packet_streams_); - for (int64 i = 0; i < num_packet_streams_; ++i) { - // Initialize latency histograms with zero counts. - packet_latencies_[i].set_num_intervals(options_.num_intervals()); - packet_latencies_[i].set_interval_size_usec(options_.interval_size_usec()); - packet_latencies_[i].mutable_counts()->Resize(options_.num_intervals(), 0); - - // Set the label for the stream. The packet labels are taken from options - // (if provided). If not, default labels are created using the input/output - // stream names. - if (labels_provided) { - packet_latencies_[i].set_label(options_.packet_labels(i)); - } else { - int64 input_stream_index = cc->Inputs().TagMap()->GetId("", i).value(); - packet_latencies_[i].set_label( - cc->Inputs().TagMap()->Names()[input_stream_index]); - } - } - - // Initialize the clock. - if (cc->InputSidePackets().HasTag(kClockTag)) { - clock_ = cc->InputSidePackets() - .Tag("CLOCK") - .Get>(); - } else { - clock_ = std::shared_ptr<::mediapipe::Clock>( - ::mediapipe::MonotonicClock::CreateSynchronizedMonotonicClock()); - } - - return absl::OkStatus(); -} - -absl::Status PacketLatencyCalculator::Process(CalculatorContext* cc) { - // Record first process timestamp if this is the first call. - if (first_process_time_usec_ < 0 && - !cc->Inputs().Tag(kReferenceSignalTag).IsEmpty()) { - first_process_time_usec_ = absl::ToUnixMicros(clock_->TimeNow()); - first_reference_timestamp_usec_ = cc->InputTimestamp().Value(); - last_reset_time_usec_ = first_process_time_usec_; - } - - if (first_process_time_usec_ < 0) { - LOG(WARNING) << "No reference packet received."; - return absl::OkStatus(); - } - - if (options_.reset_duration_usec() > 0) { - const int64 time_now_usec = absl::ToUnixMicros(clock_->TimeNow()); - if (time_now_usec - last_reset_time_usec_ >= - options_.reset_duration_usec()) { - ResetStatistics(); - last_reset_time_usec_ = time_now_usec; - } - } - - // Update latency info if there is any incoming packet. - for (int64 i = 0; i < num_packet_streams_; ++i) { - if (!cc->Inputs().Index(i).IsEmpty()) { - const auto& packet_timestamp_usec = cc->InputTimestamp().Value(); - - // Update latency statistics for this stream. - int64 current_clock_time_usec = absl::ToUnixMicros(clock_->TimeNow()); - int64 current_calibrated_timestamp_usec = - (current_clock_time_usec - first_process_time_usec_) + - first_reference_timestamp_usec_; - int64 packet_latency_usec = - current_calibrated_timestamp_usec - packet_timestamp_usec; - - // Invalid timestamps in input signals could result in negative latencies. - if (packet_latency_usec < 0) { - continue; - } - - // Update the latency, running average and histogram for this stream. - packet_latencies_[i].set_current_latency_usec(packet_latency_usec); - int64 interval_index = - packet_latency_usec / packet_latencies_[i].interval_size_usec(); - if (interval_index >= packet_latencies_[i].num_intervals()) { - interval_index = packet_latencies_[i].num_intervals() - 1; - } - packet_latencies_[i].set_counts( - interval_index, packet_latencies_[i].counts(interval_index) + 1); - sum_latencies_usec_[i] += packet_latency_usec; - num_latencies_[i] += 1; - packet_latencies_[i].set_avg_latency_usec(sum_latencies_usec_[i] / - num_latencies_[i]); - - packet_latencies_[i].set_sum_latency_usec(sum_latencies_usec_[i]); - - // Push the latency packet to output. - auto packet_latency = - absl::make_unique(packet_latencies_[i]); - cc->Outputs().Index(i).Add(packet_latency.release(), - cc->InputTimestamp()); - } - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/packet_latency_calculator.proto b/mediapipe/calculators/util/packet_latency_calculator.proto deleted file mode 100644 index 63ec5f989..000000000 --- a/mediapipe/calculators/util/packet_latency_calculator.proto +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message PacketLatencyCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional PacketLatencyCalculatorOptions ext = 172681421; - } - - // Number of intervals for the latency histogram output. - optional int64 num_intervals = 1 [default = 10]; - - // Interval size (in microseconds) for the histogram. - optional int64 interval_size_usec = 2 [default = 10000]; - - // Reset time (in microseconds) for histogram and average. The histogram and - // running average are initialized to zero periodically based on the specified - // duration. Negative value implies never resetting the statistics. - optional int64 reset_duration_usec = 3 [default = -1]; - - // Identifier labels for each input packet stream. The order of labels must - // correspond 1:1 with the input streams order. The labels are copied to the - // latency information output by the calculator. - repeated string packet_labels = 4; -} diff --git a/mediapipe/calculators/util/packet_latency_calculator_test.cc b/mediapipe/calculators/util/packet_latency_calculator_test.cc deleted file mode 100644 index aade25b0d..000000000 --- a/mediapipe/calculators/util/packet_latency_calculator_test.cc +++ /dev/null @@ -1,489 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/time/time.h" -#include "mediapipe/calculators/util/latency.pb.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/clock.h" -#include "mediapipe/framework/deps/message_matchers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/simulation_clock_executor.h" -#include "mediapipe/framework/tool/sink.h" - -namespace mediapipe { - -namespace { - -class PacketLatencyCalculatorTest : public ::testing::Test { - protected: - void SetupSimulationClock() { - auto executor = std::make_shared(4); - simulation_clock_ = executor->GetClock(); - MP_ASSERT_OK(graph_.SetExecutor("", executor)); - } - - void InitializeSingleStreamGraph() { - graph_config_ = ParseTextProtoOrDie(R"pb( - input_stream: "delayed_packet_0" - input_stream: "camera_frames" - node { - calculator: "PacketLatencyCalculator" - input_side_packet: "CLOCK:clock" - input_stream: "delayed_packet_0" - input_stream: "REFERENCE_SIGNAL:camera_frames" - output_stream: "packet_latency_0" - options { - [mediapipe.PacketLatencyCalculatorOptions.ext] { - num_intervals: 3 - interval_size_usec: 4 - reset_duration_usec: 100 - packet_labels: "dummy input 0" - } - } - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - } - )pb"); - - mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_, - &out_0_packets_); - - // Create the simulation clock side packet. - SetupSimulationClock(); - std::map side_packet; - side_packet["clock"] = - ::mediapipe::MakePacket>( - simulation_clock_); - - // Start graph run. - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - MP_ASSERT_OK(graph_.StartRun(side_packet)); - // Let Calculator::Open() calls finish before continuing. - MP_ASSERT_OK(graph_.WaitUntilIdle()); - } - - void InitializeMultipleStreamGraph() { - graph_config_ = ParseTextProtoOrDie(R"pb( - input_stream: "delayed_packet_0" - input_stream: "delayed_packet_1" - input_stream: "delayed_packet_2" - input_stream: "camera_frames" - node { - calculator: "PacketLatencyCalculator" - input_side_packet: "CLOCK:clock" - input_stream: "delayed_packet_0" - input_stream: "delayed_packet_1" - input_stream: "delayed_packet_2" - input_stream: "REFERENCE_SIGNAL:camera_frames" - output_stream: "packet_latency_0" - output_stream: "packet_latency_1" - output_stream: "packet_latency_2" - options { - [mediapipe.PacketLatencyCalculatorOptions.ext] { - num_intervals: 3 - interval_size_usec: 4 - packet_labels: "dummy input 0" - packet_labels: "dummy input 1" - packet_labels: "dummy input 2" - } - } - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - } - )pb"); - - mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_, - &out_0_packets_); - mediapipe::tool::AddVectorSink("packet_latency_1", &graph_config_, - &out_1_packets_); - mediapipe::tool::AddVectorSink("packet_latency_2", &graph_config_, - &out_2_packets_); - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - - // Create the simulation clock side packet. - simulation_clock_.reset(new SimulationClock()); - std::map side_packet; - side_packet["clock"] = - ::mediapipe::MakePacket>( - simulation_clock_); - - // Start graph run. - MP_ASSERT_OK(graph_.StartRun(side_packet)); - // Let Calculator::Open() calls finish before continuing. - MP_ASSERT_OK(graph_.WaitUntilIdle()); - } - - void InitializeSingleStreamGraphWithoutClock() { - graph_config_ = ParseTextProtoOrDie(R"pb( - input_stream: "delayed_packet_0" - input_stream: "camera_frames" - node { - calculator: "PacketLatencyCalculator" - input_stream: "delayed_packet_0" - input_stream: "REFERENCE_SIGNAL:camera_frames" - output_stream: "packet_latency_0" - options { - [mediapipe.PacketLatencyCalculatorOptions.ext] { - num_intervals: 3 - interval_size_usec: 4 - packet_labels: "dummy input 0" - } - } - input_stream_handler { - input_stream_handler: "ImmediateInputStreamHandler" - } - } - )pb"); - - mediapipe::tool::AddVectorSink("packet_latency_0", &graph_config_, - &out_0_packets_); - - // Create the simulation clock side packet. - SetupSimulationClock(); - std::map side_packet; - side_packet["clock"] = - ::mediapipe::MakePacket>( - simulation_clock_); - - // Start graph run. - MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); - MP_ASSERT_OK(graph_.StartRun(side_packet)); - // Let Calculator::Open() calls finish before continuing. - MP_ASSERT_OK(graph_.WaitUntilIdle()); - } - - PacketLatency CreatePacketLatency(const double latency_usec, - const int64 num_intervals, - const int64 interval_size_usec, - const std::vector& counts, - const int64 avg_latency_usec, - const std::string& label) { - PacketLatency latency_info; - latency_info.set_current_latency_usec(latency_usec); - latency_info.set_num_intervals(num_intervals); - latency_info.set_interval_size_usec(interval_size_usec); - int sum_counts = 0; - for (const int& count : counts) { - latency_info.add_counts(count); - sum_counts += count; - } - latency_info.set_avg_latency_usec(avg_latency_usec); - latency_info.set_sum_latency_usec(avg_latency_usec * sum_counts); - latency_info.set_label(label); - return latency_info; - } - - std::shared_ptr<::mediapipe::Clock> simulation_clock_; - CalculatorGraphConfig graph_config_; - CalculatorGraph graph_; - std::vector out_0_packets_; - std::vector out_1_packets_; - std::vector out_2_packets_; -}; - -// Calculator must not output any latency until input packets are received. -TEST_F(PacketLatencyCalculatorTest, DoesNotOutputUntilInputPacketReceived) { - // Initialize graph_. - InitializeSingleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send reference packets with timestamps 0, 6 and 10 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(0)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(6)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(10)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect zero output packets. - ASSERT_EQ(out_0_packets_.size(), 0); -} - -// Calculator must output correct latency for single stream. -TEST_F(PacketLatencyCalculatorTest, OutputsCorrectLatencyForSingleStream) { - // Initialize graph_. - InitializeSingleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send a reference packet with timestamp 10 usec at time 12 usec. - simulation_clock_->Sleep(absl::Microseconds(12)); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(10)))); - - // Add two delayed packets with timestamp 1 and 8 resp. - simulation_clock_->Sleep(absl::Microseconds(1)); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(1)))); - simulation_clock_->Sleep(absl::Microseconds(1)); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(8)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect two latency packets with timestamp 1 and 8 resp. - ASSERT_EQ(out_0_packets_.size(), 2); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 1); - EXPECT_EQ(out_0_packets_[1].Timestamp().Value(), 8); - - EXPECT_THAT( - out_0_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/10, - /*num_intervals=*/3, /*interval_size_usec=*/4, - /*counts=*/{0, 0, 1}, /*avg_latency_usec=*/10, "dummy input 0"))); - - EXPECT_THAT( - out_0_packets_[1].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/4, - /*num_intervals=*/3, /*interval_size_usec=*/4, - /*counts=*/{0, 1, 1}, /*avg_latency_usec=*/7, "dummy input 0"))); -} - -// Calculator must not output latency until reference signal is received. -TEST_F(PacketLatencyCalculatorTest, DoesNotOutputUntilReferencePacketReceived) { - // Initialize graph_. - InitializeSingleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Add two packets with timestamp 1 and 2. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(1)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(2)))); - - // Send a reference packet with timestamp 10 usec. - simulation_clock_->Sleep(absl::Microseconds(1)); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(10)))); - simulation_clock_->Sleep(absl::Microseconds(1)); - - // Add two delayed packets with timestamp 7 and 9 resp. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(7)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(9)))); - simulation_clock_->Sleep(absl::Microseconds(1)); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect two latency packets with timestamp 7 and 9 resp. The packets with - // timestamps 1 and 2 should not have any latency associated with them since - // reference signal was not sent until then. - ASSERT_EQ(out_0_packets_.size(), 2); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 7); - EXPECT_EQ(out_0_packets_[1].Timestamp().Value(), 9); - - EXPECT_THAT( - out_0_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/4, - /*num_intervals=*/3, /*interval_size_usec=*/4, - /*counts=*/{0, 1, 0}, /*avg_latency_usec=*/4, "dummy input 0"))); - - EXPECT_THAT( - out_0_packets_[1].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/2, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{1, 1, 0}, /*avg_latency_usec=*/3, "dummy input 0"))); -} - -// Calculator outputs latency even when a clock is not provided. -TEST_F(PacketLatencyCalculatorTest, OutputsCorrectLatencyWhenNoClock) { - // Initialize graph_. - InitializeSingleStreamGraphWithoutClock(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send a reference packet with timestamp 10 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(10)))); - - // Add two delayed packets with timestamp 5 and 10 resp. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(5)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(10)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect two latency packets with timestamp 5 and 10 resp. - ASSERT_EQ(out_0_packets_.size(), 2); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 5); - EXPECT_EQ(out_0_packets_[1].Timestamp().Value(), 10); -} - -// Calculator must output correct histograms counts for the corner bins. -TEST_F(PacketLatencyCalculatorTest, - OutputsCorrectLatencyStatisticsInTimeWindow) { - // Initialize graph_. - InitializeSingleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send a reference packet with timestamp 20 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(20)))); - - // Add two delayed packets with timestamp 0 and 20 resp. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(0)))); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(20)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect two latency packets with timestamp 0 and 20 resp. - ASSERT_EQ(out_0_packets_.size(), 2); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 0); - EXPECT_EQ(out_0_packets_[1].Timestamp().Value(), 20); - - EXPECT_THAT( - out_0_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/20, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{0, 0, 1}, /*avg_latency_usec=*/20, "dummy input 0"))); - - EXPECT_THAT( - out_0_packets_[1].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/0, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{1, 0, 1}, /*avg_latency_usec=*/10, "dummy input 0"))); -} - -// Calculator must reset histogram and average after specified duration. -TEST_F(PacketLatencyCalculatorTest, ResetsHistogramAndAverageCorrectly) { - // Initialize graph_. - InitializeSingleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send a reference packet with timestamp 0 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(0)))); - - // Add a delayed packet with timestamp 0 usec at time 20 usec. - simulation_clock_->Sleep(absl::Microseconds(20)); - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(0)))); - - // Do a long sleep so that histogram and average are reset. - simulation_clock_->Sleep(absl::Microseconds(100)); - - // Add a delayed packet with timestamp 115 usec at time 120 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(115)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect two latency packets with timestamp 0 and 115 resp. - ASSERT_EQ(out_0_packets_.size(), 2); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 0); - EXPECT_EQ(out_0_packets_[1].Timestamp().Value(), 115); - - EXPECT_THAT( - out_0_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/20, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{0, 0, 1}, /*avg_latency_usec=*/20, "dummy input 0"))); - - // The new average and histogram should ignore the previous latency because - // reset has happened. - EXPECT_THAT( - out_0_packets_[1].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/5, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{0, 1, 0}, /*avg_latency_usec=*/5, "dummy input 0"))); -} - -// Calculator must output correct latency for multiple streams. -TEST_F(PacketLatencyCalculatorTest, OutputsCorrectLatencyForMultipleStreams) { - // Initialize graph. - InitializeMultipleStreamGraph(); - dynamic_cast(&*simulation_clock_)->ThreadStart(); - - // Send a reference packet with timestamp 10 usec. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "camera_frames", Adopt(new double()).At(Timestamp(10)))); - - // Add delayed packets on each input stream. - - // Fastest stream. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_0", Adopt(new double()).At(Timestamp(10)))); - - // Slow stream. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_1", Adopt(new double()).At(Timestamp(5)))); - - // Slowest stream. - MP_ASSERT_OK(graph_.AddPacketToInputStream( - "delayed_packet_2", Adopt(new double()).At(Timestamp(0)))); - - dynamic_cast(&*simulation_clock_)->ThreadFinish(); - MP_ASSERT_OK(graph_.CloseAllInputStreams()); - MP_ASSERT_OK(graph_.WaitUntilDone()); - - // Expect one latency packet on each output stream. - ASSERT_EQ(out_0_packets_.size(), 1); - ASSERT_EQ(out_1_packets_.size(), 1); - ASSERT_EQ(out_2_packets_.size(), 1); - EXPECT_EQ(out_0_packets_[0].Timestamp().Value(), 10); - EXPECT_EQ(out_1_packets_[0].Timestamp().Value(), 5); - EXPECT_EQ(out_2_packets_[0].Timestamp().Value(), 0); - - EXPECT_THAT( - out_0_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/0, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{1, 0, 0}, /*avg_latency_usec=*/0, "dummy input 0"))); - EXPECT_THAT( - out_1_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/5, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{0, 1, 0}, /*avg_latency_usec=*/5, "dummy input 1"))); - EXPECT_THAT( - out_2_packets_[0].Get(), - EqualsProto(CreatePacketLatency( - /*latency_usec=*/10, /*num_intervals=*/3, - /*interval_size_usec=*/4, - /*counts=*/{0, 0, 1}, /*avg_latency_usec=*/10, "dummy input 2"))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_projection_calculator.cc b/mediapipe/calculators/util/rect_projection_calculator.cc deleted file mode 100644 index dcc6e7391..000000000 --- a/mediapipe/calculators/util/rect_projection_calculator.cc +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kNormReferenceRectTag[] = "NORM_REFERENCE_RECT"; - -} // namespace - -// Projects rectangle from reference coordinate system (defined by reference -// rectangle) to original coordinate system (in which this reference rectangle -// is defined). -// -// Inputs: -// NORM_RECT - A NormalizedRect to be projected. -// NORM_REFERENCE_RECT - A NormalizedRect that represents reference coordinate -// system for NORM_RECT and is defined in original coordinates. -// -// Outputs: -// NORM_RECT: A NormalizedRect projected to the original coordinates. -// -// Example config: -// node { -// calculator: "RectProjectionCalculator" -// input_stream: "NORM_RECT:face_rect" -// input_stream: "NORM_REFERENCE_RECT:face_reference_rect" -// output_stream: "NORM_RECT:projected_face_rect" -// } -// -class RectProjectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(RectProjectionCalculator); - -absl::Status RectProjectionCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kNormRectTag).Set(); - cc->Inputs().Tag(kNormReferenceRectTag).Set(); - cc->Outputs().Tag(kNormRectTag).Set(); - return absl::OkStatus(); -} - -absl::Status RectProjectionCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); -} - -absl::Status RectProjectionCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().Tag(kNormRectTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& rect = cc->Inputs().Tag(kNormRectTag).Get(); - const auto& reference_rect = - cc->Inputs().Tag(kNormReferenceRectTag).Get(); - - // Project center. - const float x = rect.x_center() - 0.5f; - const float y = rect.y_center() - 0.5f; - const float angle = reference_rect.rotation(); - float new_x = std::cos(angle) * x - std::sin(angle) * y; - float new_y = std::sin(angle) * x + std::cos(angle) * y; - new_x = new_x * reference_rect.width() + reference_rect.x_center(); - new_y = new_y * reference_rect.height() + reference_rect.y_center(); - - // Project size. - const float new_width = rect.width() * reference_rect.width(); - const float new_height = rect.height() * reference_rect.height(); - - // Project rotation. - const float new_rotation = rect.rotation() + reference_rect.rotation(); - - auto new_rect = absl::make_unique(); - new_rect->set_x_center(new_x); - new_rect->set_y_center(new_y); - new_rect->set_width(new_width); - new_rect->set_height(new_height); - new_rect->set_rotation(new_rotation); - - cc->Outputs().Tag(kNormRectTag).Add(new_rect.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_to_render_data_calculator.cc b/mediapipe/calculators/util/rect_to_render_data_calculator.cc deleted file mode 100644 index 3b395818f..000000000 --- a/mediapipe/calculators/util/rect_to_render_data_calculator.cc +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kRectTag[] = "RECT"; -constexpr char kNormRectsTag[] = "NORM_RECTS"; -constexpr char kRectsTag[] = "RECTS"; -constexpr char kRenderDataTag[] = "RENDER_DATA"; - -RenderAnnotation::Rectangle* NewRect( - const RectToRenderDataCalculatorOptions& options, RenderData* render_data) { - auto* annotation = render_data->add_render_annotations(); - annotation->mutable_color()->set_r(options.color().r()); - annotation->mutable_color()->set_g(options.color().g()); - annotation->mutable_color()->set_b(options.color().b()); - annotation->set_thickness(options.thickness()); - - return options.oval() ? options.filled() - ? annotation->mutable_filled_oval() - ->mutable_oval() - ->mutable_rectangle() - : annotation->mutable_oval()->mutable_rectangle() - : options.filled() - ? annotation->mutable_filled_rectangle()->mutable_rectangle() - : annotation->mutable_rectangle(); -} - -void SetRect(bool normalized, double xmin, double ymin, double width, - double height, double rotation, - RenderAnnotation::Rectangle* rect) { - if (rotation == 0.0) { - if (xmin + width < 0.0 || ymin + height < 0.0) return; - if (normalized) { - if (xmin > 1.0 || ymin > 1.0) return; - } - } - rect->set_normalized(normalized); - rect->set_left(xmin); - rect->set_top(ymin); - rect->set_right(xmin + width); - rect->set_bottom(ymin + height); - rect->set_rotation(rotation); -} - -} // namespace - -// Generates render data needed to render a rectangle in -// AnnotationOverlayCalculator. -// -// Input: -// One of the following: -// NORM_RECT: A NormalizedRect -// RECT: A Rect -// NORM_RECTS: An std::vector -// RECTS: An std::vector -// -// Output: -// RENDER_DATA: A RenderData -// -// Example config: -// node { -// calculator: "RectToRenderDataCalculator" -// input_stream: "NORM_RECT:rect" -// output_stream: "RENDER_DATA:rect_render_data" -// options: { -// [mediapipe.RectToRenderDataCalculatorOptions.ext] { -// filled: true -// color { r: 255 g: 0 b: 0 } -// thickness: 4.0 -// } -// } -// } -class RectToRenderDataCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - RectToRenderDataCalculatorOptions options_; -}; -REGISTER_CALCULATOR(RectToRenderDataCalculator); - -absl::Status RectToRenderDataCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ((cc->Inputs().HasTag(kNormRectTag) ? 1 : 0) + - (cc->Inputs().HasTag(kRectTag) ? 1 : 0) + - (cc->Inputs().HasTag(kNormRectsTag) ? 1 : 0) + - (cc->Inputs().HasTag(kRectsTag) ? 1 : 0), - 1) - << "Exactly one of NORM_RECT, RECT, NORM_RECTS or RECTS input stream " - "should be provided."; - RET_CHECK(cc->Outputs().HasTag(kRenderDataTag)); - - if (cc->Inputs().HasTag(kNormRectTag)) { - cc->Inputs().Tag(kNormRectTag).Set(); - } - if (cc->Inputs().HasTag(kRectTag)) { - cc->Inputs().Tag(kRectTag).Set(); - } - if (cc->Inputs().HasTag(kNormRectsTag)) { - cc->Inputs().Tag(kNormRectsTag).Set>(); - } - if (cc->Inputs().HasTag(kRectsTag)) { - cc->Inputs().Tag(kRectsTag).Set>(); - } - cc->Outputs().Tag(kRenderDataTag).Set(); - - return absl::OkStatus(); -} - -absl::Status RectToRenderDataCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - - return absl::OkStatus(); -} - -absl::Status RectToRenderDataCalculator::Process(CalculatorContext* cc) { - auto render_data = absl::make_unique(); - - if (cc->Inputs().HasTag(kNormRectTag) && - !cc->Inputs().Tag(kNormRectTag).IsEmpty()) { - const auto& rect = cc->Inputs().Tag(kNormRectTag).Get(); - auto* rectangle = NewRect(options_, render_data.get()); - SetRect(/*normalized=*/true, rect.x_center() - rect.width() / 2.f, - rect.y_center() - rect.height() / 2.f, rect.width(), rect.height(), - rect.rotation(), rectangle); - } - if (cc->Inputs().HasTag(kRectTag) && !cc->Inputs().Tag(kRectTag).IsEmpty()) { - const auto& rect = cc->Inputs().Tag(kRectTag).Get(); - auto* rectangle = NewRect(options_, render_data.get()); - SetRect(/*normalized=*/false, rect.x_center() - rect.width() / 2.f, - rect.y_center() - rect.height() / 2.f, rect.width(), rect.height(), - rect.rotation(), rectangle); - } - if (cc->Inputs().HasTag(kNormRectsTag) && - !cc->Inputs().Tag(kNormRectsTag).IsEmpty()) { - const auto& rects = - cc->Inputs().Tag(kNormRectsTag).Get>(); - for (auto& rect : rects) { - auto* rectangle = NewRect(options_, render_data.get()); - SetRect(/*normalized=*/true, rect.x_center() - rect.width() / 2.f, - rect.y_center() - rect.height() / 2.f, rect.width(), - rect.height(), rect.rotation(), rectangle); - } - } - if (cc->Inputs().HasTag(kRectsTag) && - !cc->Inputs().Tag(kRectsTag).IsEmpty()) { - const auto& rects = cc->Inputs().Tag(kRectsTag).Get>(); - for (auto& rect : rects) { - auto* rectangle = NewRect(options_, render_data.get()); - SetRect(/*normalized=*/false, rect.x_center() - rect.width() / 2.f, - rect.y_center() - rect.height() / 2.f, rect.width(), - rect.height(), rect.rotation(), rectangle); - } - } - - cc->Outputs() - .Tag(kRenderDataTag) - .Add(render_data.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_to_render_data_calculator.proto b/mediapipe/calculators/util/rect_to_render_data_calculator.proto deleted file mode 100644 index 9b6d5e6ee..000000000 --- a/mediapipe/calculators/util/rect_to_render_data_calculator.proto +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -message RectToRenderDataCalculatorOptions { - extend CalculatorOptions { - optional RectToRenderDataCalculatorOptions ext = 262270380; - } - - // Whether the rendered rectangle should be filled. - optional bool filled = 1; - - // Line color or filled color of the rectangle. - optional Color color = 2; - - // Thickness of the line (applicable when the rectangle is not filled). - optional double thickness = 3 [default = 1.0]; - - // Whether the rendered rectangle should be an oval. - optional bool oval = 4 [default = false]; -} diff --git a/mediapipe/calculators/util/rect_to_render_scale_calculator.cc b/mediapipe/calculators/util/rect_to_render_scale_calculator.cc deleted file mode 100644 index d94615228..000000000 --- a/mediapipe/calculators/util/rect_to_render_scale_calculator.cc +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/rect_to_render_scale_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kImageSizeTag[] = "IMAGE_SIZE"; -constexpr char kRenderScaleTag[] = "RENDER_SCALE"; - -} // namespace - -// A calculator to get scale for RenderData primitives. -// -// This calculator allows you to make RenderData primitives size (configured via -// `thickness`) to depend on actual size of the object they should highlight -// (e.g. pose, hand or face). It will give you bigger rendered primitives for -// bigger/closer objects and smaller primitives for smaller/far objects. -// -// IMPORTANT NOTE: RenderData primitives are rendered via OpenCV, which accepts -// only integer thickness. So when object goes further/closer you'll see 1 pixel -// jumps. -// -// Check `mediapipe/util/render_data.proto` for details on -// RenderData primitives and `thickness` parameter. -// -// Inputs: -// NORM_RECT: Normalized rectangle to compute object size from as maximum of -// width and height. -// IMAGE_SIZE: A std::pair represention of image width and height to -// transform normalized object width and height to absolute pixel -// coordinates. -// -// Outputs: -// RENDER_SCALE: Float value that should be used to scale RenderData -// primitives calculated as `rect_size * multiplier`. -// -// Example config: -// node { -// calculator: "RectToRenderScaleCalculator" -// input_stream: "NORM_RECT:pose_landmarks_rect" -// input_stream: "IMAGE_SIZE:image_size" -// output_stream: "RENDER_SCALE:render_scale" -// options: { -// [mediapipe.RectToRenderScaleCalculatorOptions.ext] { -// multiplier: 0.001 -// } -// } -// } -class RectToRenderScaleCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - RectToRenderScaleCalculatorOptions options_; -}; -REGISTER_CALCULATOR(RectToRenderScaleCalculator); - -absl::Status RectToRenderScaleCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kNormRectTag).Set(); - cc->Inputs().Tag(kImageSizeTag).Set>(); - cc->Outputs().Tag(kRenderScaleTag).Set(); - - return absl::OkStatus(); -} - -absl::Status RectToRenderScaleCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - options_ = cc->Options(); - - return absl::OkStatus(); -} - -absl::Status RectToRenderScaleCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().Tag(kNormRectTag).IsEmpty()) { - cc->Outputs() - .Tag(kRenderScaleTag) - .AddPacket( - MakePacket(options_.multiplier()).At(cc->InputTimestamp())); - return absl::OkStatus(); - } - - // Get image size. - int image_width; - int image_height; - std::tie(image_width, image_height) = - cc->Inputs().Tag(kImageSizeTag).Get>(); - - // Get rect size in absolute pixel coordinates. - const auto& rect = cc->Inputs().Tag(kNormRectTag).Get(); - const float rect_width = rect.width() * image_width; - const float rect_height = rect.height() * image_height; - - // Calculate render scale. - const float rect_size = std::max(rect_width, rect_height); - const float render_scale = rect_size * options_.multiplier(); - - cc->Outputs() - .Tag(kRenderScaleTag) - .AddPacket(MakePacket(render_scale).At(cc->InputTimestamp())); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_to_render_scale_calculator.proto b/mediapipe/calculators/util/rect_to_render_scale_calculator.proto deleted file mode 100644 index dda6e2c9c..000000000 --- a/mediapipe/calculators/util/rect_to_render_scale_calculator.proto +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message RectToRenderScaleCalculatorOptions { - extend CalculatorOptions { - optional RectToRenderScaleCalculatorOptions ext = 299463409; - } - - // Multiplier to apply to the rect size. - // If one defined `thickness` for RenderData primitives for object (e.g. pose, - // hand or face) of size `A` then multiplier should be `1/A`. It means that - // when actual object size on the image will be `B`, than all RenderData - // primitives will be scaled with factor `B/A`. - optional float multiplier = 1 [default = 0.01]; -} diff --git a/mediapipe/calculators/util/rect_transformation_calculator.cc b/mediapipe/calculators/util/rect_transformation_calculator.cc deleted file mode 100644 index 7c71dd5a1..000000000 --- a/mediapipe/calculators/util/rect_transformation_calculator.cc +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include - -#include "mediapipe/calculators/util/rect_transformation_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kNormRectsTag[] = "NORM_RECTS"; -constexpr char kRectTag[] = "RECT"; -constexpr char kRectsTag[] = "RECTS"; -constexpr char kImageSizeTag[] = "IMAGE_SIZE"; - -// Wraps around an angle in radians to within -M_PI and M_PI. -inline float NormalizeRadians(float angle) { - return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI)); -} - -} // namespace - -// Performs geometric transformation to the input Rect or NormalizedRect, -// correpsonding to input stream RECT or NORM_RECT respectively. When the input -// is NORM_RECT, an addition input stream IMAGE_SIZE is required, which is a -// std::pair representing the image width and height. -// -// Example config: -// node { -// calculator: "RectTransformationCalculator" -// input_stream: "NORM_RECT:rect" -// input_stream: "IMAGE_SIZE:image_size" -// output_stream: "output_rect" -// options: { -// [mediapipe.RectTransformationCalculatorOptions.ext] { -// scale_x: 2.6 -// scale_y: 2.6 -// shift_y: -0.5 -// square_long: true -// } -// } -// } -class RectTransformationCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - RectTransformationCalculatorOptions options_; - - float ComputeNewRotation(float rotation); - void TransformRect(Rect* rect); - void TransformNormalizedRect(NormalizedRect* rect, int image_width, - int image_height); -}; -REGISTER_CALCULATOR(RectTransformationCalculator); - -absl::Status RectTransformationCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK_EQ((cc->Inputs().HasTag(kNormRectTag) ? 1 : 0) + - (cc->Inputs().HasTag(kNormRectsTag) ? 1 : 0) + - (cc->Inputs().HasTag(kRectTag) ? 1 : 0) + - (cc->Inputs().HasTag(kRectsTag) ? 1 : 0), - 1); - if (cc->Inputs().HasTag(kRectTag)) { - cc->Inputs().Tag(kRectTag).Set(); - cc->Outputs().Index(0).Set(); - } - if (cc->Inputs().HasTag(kRectsTag)) { - cc->Inputs().Tag(kRectsTag).Set>(); - cc->Outputs().Index(0).Set>(); - } - if (cc->Inputs().HasTag(kNormRectTag)) { - RET_CHECK(cc->Inputs().HasTag(kImageSizeTag)); - cc->Inputs().Tag(kNormRectTag).Set(); - cc->Inputs().Tag(kImageSizeTag).Set>(); - cc->Outputs().Index(0).Set(); - } - if (cc->Inputs().HasTag(kNormRectsTag)) { - RET_CHECK(cc->Inputs().HasTag(kImageSizeTag)); - cc->Inputs().Tag(kNormRectsTag).Set>(); - cc->Inputs().Tag(kImageSizeTag).Set>(); - cc->Outputs().Index(0).Set>(); - } - - return absl::OkStatus(); -} - -absl::Status RectTransformationCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - options_ = cc->Options(); - RET_CHECK(!(options_.has_rotation() && options_.has_rotation_degrees())); - RET_CHECK(!(options_.has_square_long() && options_.has_square_short())); - - return absl::OkStatus(); -} - -absl::Status RectTransformationCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kRectTag) && !cc->Inputs().Tag(kRectTag).IsEmpty()) { - auto rect = cc->Inputs().Tag(kRectTag).Get(); - TransformRect(&rect); - cc->Outputs().Index(0).AddPacket( - MakePacket(rect).At(cc->InputTimestamp())); - } - if (cc->Inputs().HasTag(kRectsTag) && - !cc->Inputs().Tag(kRectsTag).IsEmpty()) { - auto rects = cc->Inputs().Tag(kRectsTag).Get>(); - auto output_rects = absl::make_unique>(rects.size()); - for (int i = 0; i < rects.size(); ++i) { - output_rects->at(i) = rects[i]; - auto it = output_rects->begin() + i; - TransformRect(&(*it)); - } - cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp()); - } - if (cc->Inputs().HasTag(kNormRectTag) && - !cc->Inputs().Tag(kNormRectTag).IsEmpty()) { - auto rect = cc->Inputs().Tag(kNormRectTag).Get(); - const auto& image_size = - cc->Inputs().Tag(kImageSizeTag).Get>(); - TransformNormalizedRect(&rect, image_size.first, image_size.second); - cc->Outputs().Index(0).AddPacket( - MakePacket(rect).At(cc->InputTimestamp())); - } - if (cc->Inputs().HasTag(kNormRectsTag) && - !cc->Inputs().Tag(kNormRectsTag).IsEmpty()) { - auto rects = - cc->Inputs().Tag(kNormRectsTag).Get>(); - const auto& image_size = - cc->Inputs().Tag(kImageSizeTag).Get>(); - auto output_rects = - absl::make_unique>(rects.size()); - for (int i = 0; i < rects.size(); ++i) { - output_rects->at(i) = rects[i]; - auto it = output_rects->begin() + i; - TransformNormalizedRect(&(*it), image_size.first, image_size.second); - } - cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -float RectTransformationCalculator::ComputeNewRotation(float rotation) { - if (options_.has_rotation()) { - rotation += options_.rotation(); - } else if (options_.has_rotation_degrees()) { - rotation += M_PI * options_.rotation_degrees() / 180.f; - } - return NormalizeRadians(rotation); -} - -void RectTransformationCalculator::TransformRect(Rect* rect) { - float width = rect->width(); - float height = rect->height(); - float rotation = rect->rotation(); - - if (options_.has_rotation() || options_.has_rotation_degrees()) { - rotation = ComputeNewRotation(rotation); - } - if (rotation == 0.f) { - rect->set_x_center(rect->x_center() + width * options_.shift_x()); - rect->set_y_center(rect->y_center() + height * options_.shift_y()); - } else { - const float x_shift = width * options_.shift_x() * std::cos(rotation) - - height * options_.shift_y() * std::sin(rotation); - const float y_shift = width * options_.shift_x() * std::sin(rotation) + - height * options_.shift_y() * std::cos(rotation); - rect->set_x_center(rect->x_center() + x_shift); - rect->set_y_center(rect->y_center() + y_shift); - } - - if (options_.square_long()) { - const float long_side = std::max(width, height); - width = long_side; - height = long_side; - } else if (options_.square_short()) { - const float short_side = std::min(width, height); - width = short_side; - height = short_side; - } - rect->set_width(width * options_.scale_x()); - rect->set_height(height * options_.scale_y()); -} - -void RectTransformationCalculator::TransformNormalizedRect(NormalizedRect* rect, - int image_width, - int image_height) { - float width = rect->width(); - float height = rect->height(); - float rotation = rect->rotation(); - - if (options_.has_rotation() || options_.has_rotation_degrees()) { - rotation = ComputeNewRotation(rotation); - } - if (rotation == 0.f) { - rect->set_x_center(rect->x_center() + width * options_.shift_x()); - rect->set_y_center(rect->y_center() + height * options_.shift_y()); - } else { - const float x_shift = - (image_width * width * options_.shift_x() * std::cos(rotation) - - image_height * height * options_.shift_y() * std::sin(rotation)) / - image_width; - const float y_shift = - (image_width * width * options_.shift_x() * std::sin(rotation) + - image_height * height * options_.shift_y() * std::cos(rotation)) / - image_height; - rect->set_x_center(rect->x_center() + x_shift); - rect->set_y_center(rect->y_center() + y_shift); - } - - if (options_.square_long()) { - const float long_side = - std::max(width * image_width, height * image_height); - width = long_side / image_width; - height = long_side / image_height; - } else if (options_.square_short()) { - const float short_side = - std::min(width * image_width, height * image_height); - width = short_side / image_width; - height = short_side / image_height; - } - rect->set_width(width * options_.scale_x()); - rect->set_height(height * options_.scale_y()); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_transformation_calculator.proto b/mediapipe/calculators/util/rect_transformation_calculator.proto deleted file mode 100644 index 44e781d4d..000000000 --- a/mediapipe/calculators/util/rect_transformation_calculator.proto +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message RectTransformationCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional RectTransformationCalculatorOptions ext = 262226312; - } - - // Scaling factor along the side of a rotated rect that was aligned with the - // X and Y axis before rotation respectively. - optional float scale_x = 1 [default = 1.0]; - optional float scale_y = 2 [default = 1.0]; - - // Additional rotation (counter-clockwise) around the rect center either in - // radians or in degrees. - optional float rotation = 3; - optional int32 rotation_degrees = 4; - - // Shift along the side of a rotated rect that was aligned with the X and Y - // axis before rotation respectively. The shift is relative to the length of - // corresponding side. For example, for a rect with size (0.4, 0.6), with - // shift_x = 0.5 and shift_y = -0.5 the rect is shifted along the two sides - // by 0.2 and -0.3 respectively. - optional float shift_x = 5; - optional float shift_y = 6; - - // Change the final transformed rect into a square that shares the same center - // and rotation with the rect, and with the side of the square equal to either - // the long or short side of the rect respectively. - optional bool square_long = 7; - optional bool square_short = 8; -} diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc deleted file mode 100644 index 59b21d574..000000000 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h" - -#include "mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" - -namespace mediapipe { - -namespace { - -inline float Sigmoid(float value) { return 1.0f / (1.0f + std::exp(-value)); } - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape size for heatmap tensor" << dims.size(); - } -} - -} // namespace - -namespace api2 { - -// Refines landmarks using correspond heatmap area. -// -// Input: -// NORM_LANDMARKS - Required. Input normalized landmarks to update. -// TENSORS - Required. Vector of input tensors. 0th element should be heatmap. -// The rest is unused. -// Output: -// NORM_LANDMARKS - Required. Updated normalized landmarks. -class RefineLandmarksFromHeatmapCalculatorImpl - : public NodeImpl { - public: - absl::Status Open(CalculatorContext* cc) override { return absl::OkStatus(); } - - absl::Status Process(CalculatorContext* cc) override { - // Make sure we bypass landmarks if there is no detection. - if (kInLandmarks(cc).IsEmpty()) { - return absl::OkStatus(); - } - // If for some reason heatmap is missing, just return original landmarks. - if (kInTensors(cc).IsEmpty()) { - kOutLandmarks(cc).Send(*kInLandmarks(cc)); - return absl::OkStatus(); - } - - // Check basic prerequisites. - const auto& input_tensors = *kInTensors(cc); - RET_CHECK(!input_tensors.empty()) << "Empty input tensors list. First " - "element is expeced to be a heatmap"; - - const auto& hm_tensor = input_tensors[0]; - const auto& in_lms = *kInLandmarks(cc); - auto hm_view = hm_tensor.GetCpuReadView(); - auto hm_raw = hm_view.buffer(); - const auto& options = - cc->Options(); - - ASSIGN_OR_RETURN( - auto out_lms, - RefineLandmarksFromHeatMap( - in_lms, hm_raw, hm_tensor.shape().dims, options.kernel_size(), - options.min_confidence_to_refine(), options.refine_presence(), - options.refine_visibility())); - - kOutLandmarks(cc).Send(std::move(out_lms)); - return absl::OkStatus(); - } -}; - -} // namespace api2 - -// Runs actual refinement -// High level algorithm: -// -// Heatmap is accepted as tensor in HWC layout where i-th channel is a heatmap -// for the i-th landmark. -// -// For each landmark we replace original value with a value calculated from the -// area in heatmap close to original landmark position (in particular are -// covered with kernel of size options.kernel_size). To calculate new coordinate -// from heatmap we calculate an weighted average inside the kernel. We update -// the landmark iff heatmap is confident in it's prediction i.e. max(heatmap) in -// kernel is at least options.min_confidence_to_refine big. -absl::StatusOr RefineLandmarksFromHeatMap( - const mediapipe::NormalizedLandmarkList& in_lms, - const float* heatmap_raw_data, const std::vector& heatmap_dims, - int kernel_size, float min_confidence_to_refine, bool refine_presence, - bool refine_visibility) { - ASSIGN_OR_RETURN(auto hm_dims, GetHwcFromDims(heatmap_dims)); - auto [hm_height, hm_width, hm_channels] = hm_dims; - - RET_CHECK_EQ(in_lms.landmark_size(), hm_channels) - << "Expected heatmap to have number of layers == to number of " - "landmarks"; - - int hm_row_size = hm_width * hm_channels; - int hm_pixel_size = hm_channels; - - mediapipe::NormalizedLandmarkList out_lms = in_lms; - for (int lm_index = 0; lm_index < out_lms.landmark_size(); ++lm_index) { - int center_col = out_lms.landmark(lm_index).x() * hm_width; - int center_row = out_lms.landmark(lm_index).y() * hm_height; - // Point is outside of the image let's keep it intact. - if (center_col < 0 || center_col >= hm_width || center_row < 0 || - center_col >= hm_height) { - continue; - } - - int offset = (kernel_size - 1) / 2; - // Calculate area to iterate over. Note that we decrease the kernel on - // the edges of the heatmap. Equivalent to zero border. - int begin_col = std::max(0, center_col - offset); - int end_col = std::min(hm_width, center_col + offset + 1); - int begin_row = std::max(0, center_row - offset); - int end_row = std::min(hm_height, center_row + offset + 1); - - float sum = 0; - float weighted_col = 0; - float weighted_row = 0; - float max_confidence_value = 0; - - // Main loop. Go over kernel and calculate weighted sum of coordinates, - // sum of weights and max weights. - for (int row = begin_row; row < end_row; ++row) { - for (int col = begin_col; col < end_col; ++col) { - // We expect memory to be in HWC layout without padding. - int idx = hm_row_size * row + hm_pixel_size * col + lm_index; - // Right now we hardcode sigmoid activation as it will be wasteful to - // calculate sigmoid for each value of heatmap in the model itself. If - // we ever have other activations it should be trivial to expand via - // options. - float confidence = Sigmoid(heatmap_raw_data[idx]); - sum += confidence; - max_confidence_value = std::max(max_confidence_value, confidence); - weighted_col += col * confidence; - weighted_row += row * confidence; - } - } - if (max_confidence_value >= min_confidence_to_refine && sum > 0) { - out_lms.mutable_landmark(lm_index)->set_x(weighted_col / hm_width / sum); - out_lms.mutable_landmark(lm_index)->set_y(weighted_row / hm_height / sum); - } - if (refine_presence && sum > 0 && - out_lms.landmark(lm_index).has_presence()) { - // We assume confidence in heatmaps describes landmark presence. - // If landmark is not confident in heatmaps, probably it is not present. - const float presence = out_lms.landmark(lm_index).presence(); - const float new_presence = std::min(presence, max_confidence_value); - out_lms.mutable_landmark(lm_index)->set_presence(new_presence); - } - if (refine_visibility && sum > 0 && - out_lms.landmark(lm_index).has_visibility()) { - // We assume confidence in heatmaps describes landmark presence. - // As visibility = (not occluded but still present) -> that mean that if - // landmark is not present, it is not visible as well. - // I.e. visibility confidence cannot be bigger than presence confidence. - const float visibility = out_lms.landmark(lm_index).visibility(); - const float new_visibility = std::min(visibility, max_confidence_value); - out_lms.mutable_landmark(lm_index)->set_visibility(new_visibility); - } - } - return out_lms; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h deleted file mode 100644 index 3985196bc..000000000 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_UTIL_REFINE_LANDMARKS_FROM_HEATMAP_CALCULATOR_H_ -#define MEDIAPIPE_CALCULATORS_UTIL_REFINE_LANDMARKS_FROM_HEATMAP_CALCULATOR_H_ - -#include - -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/statusor.h" - -namespace mediapipe { -namespace api2 { - -class RefineLandmarksFromHeatmapCalculator : public NodeIntf { - public: - static constexpr Input kInLandmarks{ - "NORM_LANDMARKS"}; - static constexpr Input> kInTensors{"TENSORS"}; - static constexpr Output kOutLandmarks{ - "NORM_LANDMARKS"}; - - MEDIAPIPE_NODE_INTERFACE(RefineLandmarksFromHeatmapCalculator, kInLandmarks, - kInTensors, kOutLandmarks); -}; - -} // namespace api2 - -// Exposed for testing. -absl::StatusOr RefineLandmarksFromHeatMap( - const mediapipe::NormalizedLandmarkList& in_lms, - const float* heatmap_raw_data, const std::vector& heatmap_dims, - int kernel_size, float min_confidence_to_refine, bool refine_presence, - bool refine_visibility); - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_UTIL_REFINE_LANDMARKS_FROM_HEATMAP_CALCULATOR_H_ diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto deleted file mode 100644 index eebcaed0b..000000000 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message RefineLandmarksFromHeatmapCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional RefineLandmarksFromHeatmapCalculatorOptions ext = 362281653; - } - optional int32 kernel_size = 1 [default = 9]; - optional float min_confidence_to_refine = 2 [default = 0.5]; - optional bool refine_presence = 3 [default = false]; - optional bool refine_visibility = 4 [default = false]; -} diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc deleted file mode 100644 index d484b401f..000000000 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h" - -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -mediapipe::NormalizedLandmarkList vec_to_lms( - const std::vector>& inp) { - mediapipe::NormalizedLandmarkList ret; - for (const auto& it : inp) { - auto new_lm = ret.add_landmark(); - new_lm->set_x(it.first); - new_lm->set_y(it.second); - } - return ret; -} - -std::vector> lms_to_vec( - const mediapipe::NormalizedLandmarkList& lst) { - std::vector> ret; - for (const auto& lm : lst.landmark()) { - ret.push_back({lm.x(), lm.y()}); - } - return ret; -} - -std::vector CHW_to_HWC(std::vector inp, int height, int width, - int depth) { - std::vector ret(inp.size()); - const float* inp_ptr = inp.data(); - for (int c = 0; c < depth; ++c) { - for (int row = 0; row < height; ++row) { - for (int col = 0; col < width; ++col) { - int dest_idx = width * depth * row + depth * col + c; - ret[dest_idx] = *inp_ptr; - ++inp_ptr; - } - } - } - return ret; -} - -using testing::ElementsAre; -using testing::FloatEq; -using testing::Pair; - -TEST(RefineLandmarksFromHeatmapTest, Smoke) { - float z = -10000000000000000; - // clang-format off - std::vector hm = { - z, z, z, - 1, z, z, - z, z, z}; - // clang-format on - - auto ret_or_error = RefineLandmarksFromHeatMap( - vec_to_lms({{0.5, 0.5}}), hm.data(), {3, 3, 1}, 3, 0.1, true, true); - MP_EXPECT_OK(ret_or_error); - EXPECT_THAT(lms_to_vec(*ret_or_error), - ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)))); -} - -TEST(RefineLandmarksFromHeatmapTest, MultiLayer) { - float z = -10000000000000000; - // clang-format off - std::vector hm = CHW_to_HWC({ - z, z, z, - 1, z, z, - z, z, z, - z, z, z, - 1, z, z, - z, z, z, - z, z, z, - 1, z, z, - z, z, z}, 3, 3, 3); - // clang-format on - - auto ret_or_error = RefineLandmarksFromHeatMap( - vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3, - 0.1, true, true); - MP_EXPECT_OK(ret_or_error); - EXPECT_THAT(lms_to_vec(*ret_or_error), - ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)), - Pair(FloatEq(0), FloatEq(1 / 3.)), - Pair(FloatEq(0), FloatEq(1 / 3.)))); -} - -TEST(RefineLandmarksFromHeatmapTest, KeepIfNotSure) { - float z = -10000000000000000; - // clang-format off - std::vector hm = CHW_to_HWC({ - z, z, z, - 0, z, z, - z, z, z, - z, z, z, - 0, z, z, - z, z, z, - z, z, z, - 0, z, z, - z, z, z}, 3, 3, 3); - // clang-format on - - auto ret_or_error = RefineLandmarksFromHeatMap( - vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3, - 0.6, true, true); - MP_EXPECT_OK(ret_or_error); - EXPECT_THAT(lms_to_vec(*ret_or_error), - ElementsAre(Pair(FloatEq(0.5), FloatEq(0.5)), - Pair(FloatEq(0.5), FloatEq(0.5)), - Pair(FloatEq(0.5), FloatEq(0.5)))); -} - -TEST(RefineLandmarksFromHeatmapTest, Border) { - float z = -10000000000000000; - // clang-format off - std::vector hm = CHW_to_HWC({ - z, z, z, - 0, z, 0, - z, z, z, - - z, z, z, - 0, z, 0, - z, z, 0}, 3, 3, 2); - // clang-format on - - auto ret_or_error = - RefineLandmarksFromHeatMap(vec_to_lms({{0.0, 0.0}, {0.9, 0.9}}), - hm.data(), {3, 3, 2}, 3, 0.1, true, true); - MP_EXPECT_OK(ret_or_error); - EXPECT_THAT(lms_to_vec(*ret_or_error), - ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)), - Pair(FloatEq(2 / 3.), FloatEq(1 / 6. + 2 / 6.)))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/util/set_landmark_visibility_calculator.cc b/mediapipe/calculators/util/set_landmark_visibility_calculator.cc deleted file mode 100644 index 233c3a0cb..000000000 --- a/mediapipe/calculators/util/set_landmark_visibility_calculator.cc +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kVisibilityTag[] = "VISIBILITY"; - -} // namespace - -// A calculator to set landmark visibility. -// -// Inputs: -// NORM_LANDMARKS: A NormalizedLandmarkList with only a single landmark to set -// visibility to. It's a list and not single landmark as split/concatenate -// calculators work with lists. -// -// VISIBILITY: Float visibility of the given landmark. -// -// Outputs: -// NORM_LANDMARKS: A NormalizedLandmarkList with only single landmark with -// updated visibility. -// -// Example config: -// node { -// calculator: "SetLandmarkVisibility" -// input_stream: "NORM_LANDMARKS:landmarks" -// input_stream: "VISIBILITY:visibility" -// output_stream: "NORM_LANDMARKS:landmarks_with_visibility" -// } -// -class SetLandmarkVisibilityCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(SetLandmarkVisibilityCalculator); - -absl::Status SetLandmarkVisibilityCalculator::GetContract( - CalculatorContract* cc) { - cc->Inputs().Tag(kNormalizedLandmarksTag).Set(); - cc->Inputs().Tag(kVisibilityTag).Set(); - cc->Outputs().Tag(kNormalizedLandmarksTag).Set(); - - return absl::OkStatus(); -} - -absl::Status SetLandmarkVisibilityCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); -} - -absl::Status SetLandmarkVisibilityCalculator::Process(CalculatorContext* cc) { - // Check that landmark and visibility are not empty. - // Don't emit an empty packet for this timestamp. - if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty() || - cc->Inputs().Tag(kVisibilityTag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto& in_landmarks = - cc->Inputs().Tag(kNormalizedLandmarksTag).Get(); - RET_CHECK_EQ(in_landmarks.landmark_size(), 1); - const NormalizedLandmark& in_landmark = in_landmarks.landmark(0); - - const auto& visibility = cc->Inputs().Tag(kVisibilityTag).Get(); - - auto out_landmarks = absl::make_unique(); - NormalizedLandmark* out_landmark = out_landmarks->add_landmark(); - *out_landmark = in_landmark; - // Update visibility. - out_landmark->set_visibility(visibility); - - cc->Outputs() - .Tag(kNormalizedLandmarksTag) - .Add(out_landmarks.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/thresholding_calculator.cc b/mediapipe/calculators/util/thresholding_calculator.cc deleted file mode 100644 index c86e6ca52..000000000 --- a/mediapipe/calculators/util/thresholding_calculator.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/util/thresholding_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" - -namespace mediapipe { - -// Applies a threshold on a stream of numeric values and outputs a flag and/or -// accept/reject stream. The threshold can be specified by one of the following: -// 1) Input stream. -// 2) Input side packet. -// 3) Calculator option. -// -// Input: -// FLOAT: A float, which will be cast to double to be compared with a -// threshold of double type. -// THRESHOLD(optional): A double specifying the threshold at current timestamp. -// -// Output: -// FLAG(optional): A boolean indicating if the input value is larger than the -// threshold. -// ACCEPT(optional): A packet will be sent if the value is larger than the -// threshold. -// REJECT(optional): A packet will be sent if the value is no larger than the -// threshold. -// -// Usage example: -// node { -// calculator: "ThresholdingCalculator" -// input_stream: "FLOAT:score" -// output_stream: "ACCEPT:accept" -// output_stream: "REJECT:reject" -// options: { -// [mediapipe.ThresholdingCalculatorOptions.ext] { -// threshold: 0.1 -// } -// } -// } -class ThresholdingCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - double threshold_{}; -}; -REGISTER_CALCULATOR(ThresholdingCalculator); - -absl::Status ThresholdingCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("FLOAT")); - cc->Inputs().Tag("FLOAT").Set(); - - if (cc->Outputs().HasTag("FLAG")) { - cc->Outputs().Tag("FLAG").Set(); - } - if (cc->Outputs().HasTag("ACCEPT")) { - cc->Outputs().Tag("ACCEPT").Set(); - } - if (cc->Outputs().HasTag("REJECT")) { - cc->Outputs().Tag("REJECT").Set(); - } - if (cc->Inputs().HasTag("THRESHOLD")) { - cc->Inputs().Tag("THRESHOLD").Set(); - } - if (cc->InputSidePackets().HasTag("THRESHOLD")) { - cc->InputSidePackets().Tag("THRESHOLD").Set(); - RET_CHECK(!cc->Inputs().HasTag("THRESHOLD")) - << "Using both the threshold input side packet and input stream is not " - "supported."; - } - - return absl::OkStatus(); -} - -absl::Status ThresholdingCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = - cc->Options<::mediapipe::ThresholdingCalculatorOptions>(); - if (options.has_threshold()) { - RET_CHECK(!cc->Inputs().HasTag("THRESHOLD")) - << "Using both the threshold option and input stream is not supported."; - RET_CHECK(!cc->InputSidePackets().HasTag("THRESHOLD")) - << "Using both the threshold option and input side packet is not " - "supported."; - threshold_ = options.threshold(); - } - - if (cc->InputSidePackets().HasTag("THRESHOLD")) { - threshold_ = cc->InputSidePackets().Tag("THRESHOLD").Get(); - } - return absl::OkStatus(); -} - -absl::Status ThresholdingCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag("THRESHOLD") && - !cc->Inputs().Tag("THRESHOLD").IsEmpty()) { - threshold_ = cc->Inputs().Tag("THRESHOLD").Get(); - } - - bool accept = false; - RET_CHECK(!cc->Inputs().Tag("FLOAT").IsEmpty()); - accept = - static_cast(cc->Inputs().Tag("FLOAT").Get()) > threshold_; - - if (cc->Outputs().HasTag("FLAG")) { - cc->Outputs().Tag("FLAG").AddPacket( - MakePacket(accept).At(cc->InputTimestamp())); - } - - if (accept && cc->Outputs().HasTag("ACCEPT")) { - cc->Outputs().Tag("ACCEPT").AddPacket( - MakePacket(true).At(cc->InputTimestamp())); - } - if (!accept && cc->Outputs().HasTag("REJECT")) { - cc->Outputs().Tag("REJECT").AddPacket( - MakePacket(false).At(cc->InputTimestamp())); - } - - return absl::OkStatus(); -} -} // namespace mediapipe diff --git a/mediapipe/calculators/util/thresholding_calculator.proto b/mediapipe/calculators/util/thresholding_calculator.proto deleted file mode 100644 index b8d81ad5d..000000000 --- a/mediapipe/calculators/util/thresholding_calculator.proto +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message ThresholdingCalculatorOptions { - extend CalculatorOptions { - optional ThresholdingCalculatorOptions ext = 259990498; - } - - optional double threshold = 1; -} diff --git a/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.cc b/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.cc deleted file mode 100644 index 790b426de..000000000 --- a/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.cc +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/container/node_hash_map.h" -#include "mediapipe/calculators/util/timed_box_list_id_to_label_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/resource_util.h" -#include "mediapipe/util/tracking/box_tracker.pb.h" - -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -using mediapipe::TimedBoxProto; -using mediapipe::TimedBoxProtoList; - -// Takes a label map (from label IDs to names), and populate the label field in -// TimedBoxProto according to it's ID. -// -// Example usage: -// node { -// calculator: "TimedBoxListIdToLabelCalculator" -// input_stream: "input_timed_box_list" -// output_stream: "output_timed_box_list" -// node_options: { -// [mediapipe.TimedBoxListIdToLabelCalculatorOptions] { -// label_map_path: "labelmap.txt" -// } -// } -// } -class TimedBoxListIdToLabelCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::node_hash_map label_map_; -}; -REGISTER_CALCULATOR(TimedBoxListIdToLabelCalculator); - -absl::Status TimedBoxListIdToLabelCalculator::GetContract( - CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - - return absl::OkStatus(); -} - -absl::Status TimedBoxListIdToLabelCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = - cc->Options<::mediapipe::TimedBoxListIdToLabelCalculatorOptions>(); - - std::string string_path; - ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path())); - std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - return absl::OkStatus(); -} - -absl::Status TimedBoxListIdToLabelCalculator::Process(CalculatorContext* cc) { - const auto& input_list = cc->Inputs().Index(0).Get(); - auto output_list = absl::make_unique(); - for (const auto& input_box : input_list.box()) { - TimedBoxProto* box_ptr = output_list->add_box(); - *box_ptr = input_box; - - if (label_map_.find(input_box.id()) != label_map_.end()) { - box_ptr->set_label(label_map_[input_box.id()]); - } - } - cc->Outputs().Index(0).Add(output_list.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.proto b/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.proto deleted file mode 100644 index 95cf76590..000000000 --- a/mediapipe/calculators/util/timed_box_list_id_to_label_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TimedBoxListIdToLabelCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional TimedBoxListIdToLabelCalculatorOptions ext = 297701606; - } - - // Path to a label map file for getting the actual name of detected classes. - optional string label_map_path = 1; -} diff --git a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc b/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc deleted file mode 100644 index 53c2ffa2f..000000000 --- a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "mediapipe/calculators/util/timed_box_list_to_render_data_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" -#include "mediapipe/util/tracking/box_tracker.pb.h" -#include "mediapipe/util/tracking/tracking.pb.h" - -namespace mediapipe { - -namespace { - -constexpr char kTimedBoxListTag[] = "BOX_LIST"; -constexpr char kRenderDataTag[] = "RENDER_DATA"; - -void AddTimedBoxProtoToRenderData( - const TimedBoxProto& box_proto, - const TimedBoxListToRenderDataCalculatorOptions& options, - RenderData* render_data) { - if (box_proto.has_quad() && box_proto.quad().vertices_size() > 0 && - box_proto.quad().vertices_size() % 2 == 0) { - const int num_corners = box_proto.quad().vertices_size() / 2; - for (int i = 0; i < num_corners; ++i) { - const int next_corner = (i + 1) % num_corners; - auto* line_annotation = render_data->add_render_annotations(); - line_annotation->mutable_color()->set_r(options.box_color().r()); - line_annotation->mutable_color()->set_g(options.box_color().g()); - line_annotation->mutable_color()->set_b(options.box_color().b()); - line_annotation->set_thickness(options.thickness()); - RenderAnnotation::Line* line = line_annotation->mutable_line(); - line->set_normalized(true); - line->set_x_start(box_proto.quad().vertices(i * 2)); - line->set_y_start(box_proto.quad().vertices(i * 2 + 1)); - line->set_x_end(box_proto.quad().vertices(next_corner * 2)); - line->set_y_end(box_proto.quad().vertices(next_corner * 2 + 1)); - } - } else { - auto* rect_annotation = render_data->add_render_annotations(); - rect_annotation->mutable_color()->set_r(options.box_color().r()); - rect_annotation->mutable_color()->set_g(options.box_color().g()); - rect_annotation->mutable_color()->set_b(options.box_color().b()); - rect_annotation->set_thickness(options.thickness()); - RenderAnnotation::Rectangle* rect = rect_annotation->mutable_rectangle(); - rect->set_normalized(true); - rect->set_left(box_proto.left()); - rect->set_right(box_proto.right()); - rect->set_top(box_proto.top()); - rect->set_bottom(box_proto.bottom()); - rect->set_rotation(box_proto.rotation()); - } - - if (box_proto.has_label()) { - auto* label_annotation = render_data->add_render_annotations(); - label_annotation->mutable_color()->set_r(options.box_color().r()); - label_annotation->mutable_color()->set_g(options.box_color().g()); - label_annotation->mutable_color()->set_b(options.box_color().b()); - label_annotation->set_thickness(options.thickness()); - RenderAnnotation::Text* text = label_annotation->mutable_text(); - text->set_display_text(box_proto.label()); - text->set_normalized(true); - constexpr float text_left_start = 0.2f; - text->set_left((1.0f - text_left_start) * box_proto.left() + - text_left_start * box_proto.right()); - constexpr float text_baseline = 0.6f; - text->set_baseline(text_baseline * box_proto.bottom() + - (1.0f - text_baseline) * box_proto.top()); - constexpr float text_height = 0.1f; - text->set_font_height(std::min(box_proto.bottom() - box_proto.top(), - box_proto.right() - box_proto.left()) * - text_height); - } -} - -} // namespace - -// A calculator that converts TimedBoxProtoList proto to RenderData proto for -// visualization. If the input TimedBoxProto contains `quad` field, this -// calculator will draw a quadrilateral based on it. Otherwise this calculator -// will draw a rotated rectangle based on `top`, `bottom`, `left`, `right` and -// `rotation` fields -// -// Example config: -// node { -// calculator: "TimedBoxListToRenderDataCalculator" -// input_stream: "BOX_LIST:landmarks" -// output_stream: "RENDER_DATA:render_data" -// options { -// [TimedBoxListToRenderDataCalculatorOptions.ext] { -// box_color { r: 0 g: 255 b: 0 } -// thickness: 4.0 -// } -// } -// } -class TimedBoxListToRenderDataCalculator : public CalculatorBase { - public: - TimedBoxListToRenderDataCalculator() {} - ~TimedBoxListToRenderDataCalculator() override {} - TimedBoxListToRenderDataCalculator( - const TimedBoxListToRenderDataCalculator&) = delete; - TimedBoxListToRenderDataCalculator& operator=( - const TimedBoxListToRenderDataCalculator&) = delete; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - TimedBoxListToRenderDataCalculatorOptions options_; -}; -REGISTER_CALCULATOR(TimedBoxListToRenderDataCalculator); - -absl::Status TimedBoxListToRenderDataCalculator::GetContract( - CalculatorContract* cc) { - if (cc->Inputs().HasTag(kTimedBoxListTag)) { - cc->Inputs().Tag(kTimedBoxListTag).Set(); - } - cc->Outputs().Tag(kRenderDataTag).Set(); - return absl::OkStatus(); -} - -absl::Status TimedBoxListToRenderDataCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - options_ = cc->Options(); - - return absl::OkStatus(); -} - -absl::Status TimedBoxListToRenderDataCalculator::Process( - CalculatorContext* cc) { - auto render_data = absl::make_unique(); - - if (cc->Inputs().HasTag(kTimedBoxListTag)) { - const auto& box_list = - cc->Inputs().Tag(kTimedBoxListTag).Get(); - - for (const auto& box : box_list.box()) { - AddTimedBoxProtoToRenderData(box, options_, render_data.get()); - } - } - - cc->Outputs() - .Tag(kRenderDataTag) - .Add(render_data.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.proto b/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.proto deleted file mode 100644 index 31e1ecbef..000000000 --- a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.proto +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/color.proto"; - -message TimedBoxListToRenderDataCalculatorOptions { - extend CalculatorOptions { - optional TimedBoxListToRenderDataCalculatorOptions ext = 289899854; - } - - // Color of boxes. - optional Color box_color = 1; - - // Thickness of the drawing of boxes. - optional double thickness = 2 [default = 1.0]; -} diff --git a/mediapipe/calculators/util/to_image_calculator.cc b/mediapipe/calculators/util/to_image_calculator.cc deleted file mode 100644 index 5e119fca7..000000000 --- a/mediapipe/calculators/util/to_image_calculator.cc +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" - -#if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_calculator_helper.h" -#endif // !MEDIAPIPE_DISABLE_GPU - -namespace mediapipe { - -namespace { -constexpr char kImageFrameTag[] = "IMAGE_CPU"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -constexpr char kImageTag[] = "IMAGE"; -} // namespace - -// A calculator for converting from legacy MediaPipe datatypes into a -// unified image container. -// -// Inputs: -// One of the following two tags: -// IMAGE_CPU: An ImageFrame containing input image. -// IMAGE_GPU: A GpuBuffer containing input image. -// -// Output: -// IMAGE: An Image containing output image. -// -// Note: -// No CPU/GPU conversion is done. -// -class ToImageCalculator : public CalculatorBase { - public: - ToImageCalculator() = default; - ~ToImageCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - // From Calculator. - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status RenderGpu(CalculatorContext* cc); - absl::Status RenderCpu(CalculatorContext* cc); - - bool gpu_input_ = false; - bool gpu_initialized_ = false; -#if !MEDIAPIPE_DISABLE_GPU - mediapipe::GlCalculatorHelper gpu_helper_; -#endif // !MEDIAPIPE_DISABLE_GPU -}; -REGISTER_CALCULATOR(ToImageCalculator); - -absl::Status ToImageCalculator::GetContract(CalculatorContract* cc) { - cc->Outputs().Tag(kImageTag).Set(); - - bool gpu_input = false; - - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("Cannot have multiple inputs."); - } - - if (cc->Inputs().HasTag(kGpuBufferTag)) { -#if !MEDIAPIPE_DISABLE_GPU - cc->Inputs().Tag(kGpuBufferTag).Set(); - gpu_input = true; -#else - RET_CHECK_FAIL() << "GPU is disabled. Cannot use IMAGE_GPU stream."; -#endif // !MEDIAPIPE_DISABLE_GPU - } - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } - - if (gpu_input) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); -#endif // !MEDIAPIPE_DISABLE_GPU - } - - return absl::OkStatus(); -} - -absl::Status ToImageCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kGpuBufferTag)) { - gpu_input_ = true; - } - - if (gpu_input_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); -#endif - } // !MEDIAPIPE_DISABLE_GPU - - return absl::OkStatus(); -} - -absl::Status ToImageCalculator::Process(CalculatorContext* cc) { - if (gpu_input_) { -#if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([&cc]() -> absl::Status { - auto& input = cc->Inputs().Tag(kGpuBufferTag).Get(); - // Wrap texture pointer; shallow copy. - auto output = std::make_unique(input); - cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); - })); -#endif // !MEDIAPIPE_DISABLE_GPU - } else { - // The input ImageFrame. - auto& input = cc->Inputs().Tag(kImageFrameTag).Get(); - // Make a copy of the input packet to co-own the input ImageFrame. - Packet* packet_copy_ptr = - new Packet(cc->Inputs().Tag(kImageFrameTag).Value()); - // Create an output Image that (co-)owns a new ImageFrame that points to - // the same pixel data as the input ImageFrame and also owns the packet - // copy. As a result, the output Image indirectly co-owns the input - // ImageFrame. This ensures a correct life span of the shared pixel data. - std::unique_ptr output = - std::make_unique( - std::make_shared( - input.Format(), input.Width(), input.Height(), - input.WidthStep(), const_cast(input.PixelData()), - [packet_copy_ptr](uint8*) { delete packet_copy_ptr; })); - cc->Outputs().Tag(kImageTag).Add(output.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -absl::Status ToImageCalculator::Close(CalculatorContext* cc) { - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/top_k_scores_calculator.cc b/mediapipe/calculators/util/top_k_scores_calculator.cc deleted file mode 100644 index 37d1b2ab2..000000000 --- a/mediapipe/calculators/util/top_k_scores_calculator.cc +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "absl/container/node_hash_map.h" -#include "mediapipe/calculators/util/top_k_scores_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" -#include "mediapipe/util/resource_util.h" - -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -// A calculator that takes a vector of scores and returns the indexes, scores, -// labels of the top k elements, classification protos, and summary std::string -// (in csv format). -// -// Usage example: -// node { -// calculator: "TopKScoresCalculator" -// input_stream: "SCORES:score_vector" -// output_stream: "TOP_K_INDEXES:top_k_indexes" -// output_stream: "TOP_K_SCORES:top_k_scores" -// output_stream: "TOP_K_LABELS:top_k_labels" -// output_stream: "TOP_K_CLASSIFICATIONS:top_k_classes" -// output_stream: "SUMMARY:summary" -// options: { -// [mediapipe.TopKScoresCalculatorOptions.ext] { -// top_k: 5 -// threshold: 0.1 -// label_map_path: "/path/to/label/map" -// } -// } -// } -class TopKScoresCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::Status LoadLabelmap(std::string label_map_path); - - int top_k_ = -1; - float threshold_ = 0.0; - absl::node_hash_map label_map_; - bool label_map_loaded_ = false; -}; -REGISTER_CALCULATOR(TopKScoresCalculator); - -absl::Status TopKScoresCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("SCORES")); - cc->Inputs().Tag("SCORES").Set>(); - if (cc->Outputs().HasTag("TOP_K_INDEXES")) { - cc->Outputs().Tag("TOP_K_INDEXES").Set>(); - } - if (cc->Outputs().HasTag("TOP_K_SCORES")) { - cc->Outputs().Tag("TOP_K_SCORES").Set>(); - } - if (cc->Outputs().HasTag("TOP_K_LABELS")) { - cc->Outputs().Tag("TOP_K_LABELS").Set>(); - } - if (cc->Outputs().HasTag("CLASSIFICATIONS")) { - cc->Outputs().Tag("CLASSIFICATIONS").Set(); - } - if (cc->Outputs().HasTag("SUMMARY")) { - cc->Outputs().Tag("SUMMARY").Set(); - } - return absl::OkStatus(); -} - -absl::Status TopKScoresCalculator::Open(CalculatorContext* cc) { - const auto& options = cc->Options<::mediapipe::TopKScoresCalculatorOptions>(); - RET_CHECK(options.has_top_k() || options.has_threshold()) - << "Must specify at least one of the top_k and threshold fields in " - "TopKScoresCalculatorOptions."; - if (options.has_top_k()) { - RET_CHECK(options.top_k() > 0) << "top_k must be greater than zero."; - top_k_ = options.top_k(); - } - if (options.has_threshold()) { - threshold_ = options.threshold(); - } - if (options.has_label_map_path()) { - MP_RETURN_IF_ERROR(LoadLabelmap(options.label_map_path())); - } - if (cc->Outputs().HasTag("TOP_K_LABELS")) { - RET_CHECK(!label_map_.empty()); - } - return absl::OkStatus(); -} - -absl::Status TopKScoresCalculator::Process(CalculatorContext* cc) { - const std::vector& input_vector = - cc->Inputs().Tag("SCORES").Get>(); - std::vector top_k_indexes; - - std::vector top_k_scores; - - std::vector top_k_labels; - - if (top_k_ > 0) { - top_k_indexes.reserve(top_k_); - top_k_scores.reserve(top_k_); - top_k_labels.reserve(top_k_); - } - std::priority_queue, std::vector>, - std::greater>> - pq; - for (int i = 0; i < input_vector.size(); ++i) { - if (input_vector[i] < threshold_) { - continue; - } - if (top_k_ > 0) { - if (pq.size() < top_k_) { - pq.push(std::pair(input_vector[i], i)); - } else if (pq.top().first < input_vector[i]) { - pq.pop(); - pq.push(std::pair(input_vector[i], i)); - } - } else { - pq.push(std::pair(input_vector[i], i)); - } - } - - while (!pq.empty()) { - top_k_indexes.push_back(pq.top().second); - top_k_scores.push_back(pq.top().first); - pq.pop(); - } - reverse(top_k_indexes.begin(), top_k_indexes.end()); - reverse(top_k_scores.begin(), top_k_scores.end()); - - if (label_map_loaded_) { - for (int index : top_k_indexes) { - top_k_labels.push_back(label_map_[index]); - } - } - if (cc->Outputs().HasTag("TOP_K_INDEXES")) { - cc->Outputs() - .Tag("TOP_K_INDEXES") - .AddPacket(MakePacket>(top_k_indexes) - .At(cc->InputTimestamp())); - } - if (cc->Outputs().HasTag("TOP_K_SCORES")) { - cc->Outputs() - .Tag("TOP_K_SCORES") - .AddPacket(MakePacket>(top_k_scores) - .At(cc->InputTimestamp())); - } - if (cc->Outputs().HasTag("TOP_K_LABELS")) { - cc->Outputs() - .Tag("TOP_K_LABELS") - .AddPacket(MakePacket>(top_k_labels) - .At(cc->InputTimestamp())); - } - - if (cc->Outputs().HasTag("SUMMARY")) { - std::vector results; - for (int index = 0; index < top_k_indexes.size(); ++index) { - if (label_map_loaded_) { - results.push_back( - absl::StrCat(top_k_labels[index], ":", top_k_scores[index])); - } else { - results.push_back( - absl::StrCat(top_k_indexes[index], ":", top_k_scores[index])); - } - } - cc->Outputs().Tag("SUMMARY").AddPacket( - MakePacket(absl::StrJoin(results, ",")) - .At(cc->InputTimestamp())); - } - - if (cc->Outputs().HasTag("TOP_K_CLASSIFICATION")) { - auto classification_list = absl::make_unique(); - for (int index = 0; index < top_k_indexes.size(); ++index) { - Classification* classification = - classification_list->add_classification(); - classification->set_index(top_k_indexes[index]); - classification->set_score(top_k_scores[index]); - if (label_map_loaded_) { - classification->set_label(top_k_labels[index]); - } - } - } - return absl::OkStatus(); -} - -absl::Status TopKScoresCalculator::LoadLabelmap(std::string label_map_path) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(label_map_path)); - std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); - - std::istringstream stream(label_map_string); - std::string line; - int i = 0; - while (std::getline(stream, line)) { - label_map_[i++] = line; - } - label_map_loaded_ = true; - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/top_k_scores_calculator.proto b/mediapipe/calculators/util/top_k_scores_calculator.proto deleted file mode 100644 index 08fb7a756..000000000 --- a/mediapipe/calculators/util/top_k_scores_calculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message TopKScoresCalculatorOptions { - extend CalculatorOptions { - optional TopKScoresCalculatorOptions ext = 271211788; - } - // How many highest scoring packets to output. - optional int32 top_k = 1; - - // If set, only keep the scores that are greater than the threshold. - optional float threshold = 2; - - // Path to a label map file for getting the actual name of classes. - optional string label_map_path = 3; -} diff --git a/mediapipe/calculators/util/top_k_scores_calculator_test.cc b/mediapipe/calculators/util/top_k_scores_calculator_test.cc deleted file mode 100644 index 6e6a2ebad..000000000 --- a/mediapipe/calculators/util/top_k_scores_calculator_test.cc +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -TEST(TopKScoresCalculatorTest, TestNodeConfig) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TopKScoresCalculator" - input_stream: "SCORES:score_vector" - output_stream: "TOP_K_INDEXES:top_k_indexes" - output_stream: "TOP_K_SCORES:top_k_scores" - options: { - [mediapipe.TopKScoresCalculatorOptions.ext] {} - } - )pb")); - - auto status = runner.Run(); - ASSERT_TRUE(!status.ok()); - EXPECT_THAT( - status.ToString(), - testing::HasSubstr( - "Must specify at least one of the top_k and threshold fields")); -} - -TEST(TopKScoresCalculatorTest, TestTopKOnly) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TopKScoresCalculator" - input_stream: "SCORES:score_vector" - output_stream: "TOP_K_INDEXES:top_k_indexes" - output_stream: "TOP_K_SCORES:top_k_scores" - options: { - [mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 2 } - } - )pb")); - - std::vector score_vector{0.9, 0.2, 0.3, 1.0, 0.1}; - - runner.MutableInputs()->Tag("SCORES").packets.push_back( - MakePacket>(score_vector).At(Timestamp(0))); - - MP_ASSERT_OK(runner.Run()); - const std::vector& indexes_outputs = - runner.Outputs().Tag("TOP_K_INDEXES").packets; - ASSERT_EQ(1, indexes_outputs.size()); - const auto& indexes = indexes_outputs[0].Get>(); - EXPECT_EQ(2, indexes.size()); - EXPECT_EQ(3, indexes[0]); - EXPECT_EQ(0, indexes[1]); - const std::vector& scores_outputs = - runner.Outputs().Tag("TOP_K_SCORES").packets; - ASSERT_EQ(1, scores_outputs.size()); - const auto& scores = scores_outputs[0].Get>(); - EXPECT_EQ(2, scores.size()); - EXPECT_NEAR(1, scores[0], 1e-5); - EXPECT_NEAR(0.9, scores[1], 1e-5); -} - -TEST(TopKScoresCalculatorTest, TestThresholdOnly) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TopKScoresCalculator" - input_stream: "SCORES:score_vector" - output_stream: "TOP_K_INDEXES:top_k_indexes" - output_stream: "TOP_K_SCORES:top_k_scores" - options: { - [mediapipe.TopKScoresCalculatorOptions.ext] { threshold: 0.2 } - } - )pb")); - - std::vector score_vector{0.9, 0.2, 0.3, 1.0, 0.1}; - - runner.MutableInputs()->Tag("SCORES").packets.push_back( - MakePacket>(score_vector).At(Timestamp(0))); - - MP_ASSERT_OK(runner.Run()); - const std::vector& indexes_outputs = - runner.Outputs().Tag("TOP_K_INDEXES").packets; - ASSERT_EQ(1, indexes_outputs.size()); - const auto& indexes = indexes_outputs[0].Get>(); - EXPECT_EQ(4, indexes.size()); - EXPECT_EQ(3, indexes[0]); - EXPECT_EQ(0, indexes[1]); - EXPECT_EQ(2, indexes[2]); - EXPECT_EQ(1, indexes[3]); - const std::vector& scores_outputs = - runner.Outputs().Tag("TOP_K_SCORES").packets; - ASSERT_EQ(1, scores_outputs.size()); - const auto& scores = scores_outputs[0].Get>(); - EXPECT_EQ(4, scores.size()); - EXPECT_NEAR(1.0, scores[0], 1e-5); - EXPECT_NEAR(0.9, scores[1], 1e-5); - EXPECT_NEAR(0.3, scores[2], 1e-5); - EXPECT_NEAR(0.2, scores[3], 1e-5); -} - -TEST(TopKScoresCalculatorTest, TestBothTopKAndThreshold) { - CalculatorRunner runner(ParseTextProtoOrDie(R"pb( - calculator: "TopKScoresCalculator" - input_stream: "SCORES:score_vector" - output_stream: "TOP_K_INDEXES:top_k_indexes" - output_stream: "TOP_K_SCORES:top_k_scores" - options: { - [mediapipe.TopKScoresCalculatorOptions.ext] { top_k: 4 threshold: 0.3 } - } - )pb")); - - std::vector score_vector{0.9, 0.2, 0.3, 1.0, 0.1}; - - runner.MutableInputs()->Tag("SCORES").packets.push_back( - MakePacket>(score_vector).At(Timestamp(0))); - - MP_ASSERT_OK(runner.Run()); - const std::vector& indexes_outputs = - runner.Outputs().Tag("TOP_K_INDEXES").packets; - ASSERT_EQ(1, indexes_outputs.size()); - const auto& indexes = indexes_outputs[0].Get>(); - EXPECT_EQ(3, indexes.size()); - EXPECT_EQ(3, indexes[0]); - EXPECT_EQ(0, indexes[1]); - EXPECT_EQ(2, indexes[2]); - const std::vector& scores_outputs = - runner.Outputs().Tag("TOP_K_SCORES").packets; - ASSERT_EQ(1, scores_outputs.size()); - const auto& scores = scores_outputs[0].Get>(); - EXPECT_EQ(3, scores.size()); - EXPECT_NEAR(1.0, scores[0], 1e-5); - EXPECT_NEAR(0.9, scores[1], 1e-5); - EXPECT_NEAR(0.3, scores[2], 1e-5); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/visibility_copy_calculator.cc b/mediapipe/calculators/util/visibility_copy_calculator.cc deleted file mode 100644 index f85ff9ea2..000000000 --- a/mediapipe/calculators/util/visibility_copy_calculator.cc +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2021 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/algorithm/container.h" -#include "mediapipe/calculators/util/visibility_copy_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { - -namespace { - -constexpr char kLandmarksFromTag[] = "LANDMARKS_FROM"; -constexpr char kNormalizedLandmarksFromTag[] = "NORM_LANDMARKS_FROM"; -constexpr char kLandmarksToTag[] = "LANDMARKS_TO"; -constexpr char kNormalizedLandmarksToTag[] = "NORM_LANDMARKS_TO"; - -} // namespace - -// A calculator to copy visibility and presence between landmarks. -// -// Landmarks to copy from and to copy to can be of different type (normalized or -// non-normalized), but ladnmarks to copy to and output landmarks should be of -// the same type. Exactly one stream to copy landmarks from, to copy to and to -// output should be provided. -// -// Inputs: -// LANDMARKS_FROM (optional): A LandmarkList of landmarks to copy from. -// NORM_LANDMARKS_FROM (optional): A NormalizedLandmarkList of landmarks to -// copy from. -// LANDMARKS_TO (optional): A LandmarkList of landmarks to copy to. -// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy -// to. -// -// Outputs: -// LANDMARKS_TO (optional): A LandmarkList of landmarks from LANDMARKS_TO and -// visibility/presence from LANDMARKS_FROM or NORM_LANDMARKS_FROM. -// NORM_LANDMARKS_TO (optional): A NormalizedLandmarkList of landmarks to copy -// to. -// -// Example config: -// node { -// calculator: "VisibilityCopyCalculator" -// input_stream: "NORM_LANDMARKS_FROM:pose_landmarks" -// input_stream: "LANDMARKS_TO:pose_world_landmarks" -// output_stream: "LANDMARKS_TO:pose_world_landmarks_with_visibility" -// options: { -// [mediapipe.VisibilityCopyCalculatorOptions.ext] { -// copy_visibility: true -// copy_presence: true -// } -// } -// } -// -class VisibilityCopyCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - template - absl::Status CopyVisibility(CalculatorContext* cc, - const std::string& landmarks_from_tag, - const std::string& landmarks_to_tag); - - bool copy_visibility_; - bool copy_presence_; -}; -REGISTER_CALCULATOR(VisibilityCopyCalculator); - -absl::Status VisibilityCopyCalculator::GetContract(CalculatorContract* cc) { - // Landmarks to copy from. - RET_CHECK(cc->Inputs().HasTag(kLandmarksFromTag) ^ - cc->Inputs().HasTag(kNormalizedLandmarksFromTag)) - << "Exatly one landmarks stream to copy from should be provided"; - if (cc->Inputs().HasTag(kLandmarksFromTag)) { - cc->Inputs().Tag(kLandmarksFromTag).Set(); - } else { - cc->Inputs().Tag(kNormalizedLandmarksFromTag).Set(); - } - - // Landmarks to copy to and corresponding output landmarks. - RET_CHECK(cc->Inputs().HasTag(kLandmarksToTag) ^ - cc->Inputs().HasTag(kNormalizedLandmarksToTag)) - << "Exatly one landmarks stream to copy to should be provided"; - if (cc->Inputs().HasTag(kLandmarksToTag)) { - cc->Inputs().Tag(kLandmarksToTag).Set(); - - RET_CHECK(cc->Outputs().HasTag(kLandmarksToTag)) - << "Landmarks to copy to and output stream types should be the same"; - cc->Outputs().Tag(kLandmarksToTag).Set(); - } else { - cc->Inputs().Tag(kNormalizedLandmarksToTag).Set(); - - RET_CHECK(cc->Outputs().HasTag(kNormalizedLandmarksToTag)) - << "Landmarks to copy to and output stream types should be the same"; - cc->Outputs().Tag(kNormalizedLandmarksToTag).Set(); - } - - return absl::OkStatus(); -} - -absl::Status VisibilityCopyCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - const auto& options = cc->Options(); - copy_visibility_ = options.copy_visibility(); - copy_presence_ = options.copy_presence(); - - return absl::OkStatus(); -} - -absl::Status VisibilityCopyCalculator::Process(CalculatorContext* cc) { - // Switch between all four possible combinations of landmarks from and - // landmarks to types (normalized and non-normalized). - auto status = absl::OkStatus(); - if (cc->Inputs().HasTag(kLandmarksFromTag)) { - if (cc->Inputs().HasTag(kLandmarksToTag)) { - status = CopyVisibility(cc, kLandmarksFromTag, - kLandmarksToTag); - } else { - status = CopyVisibility( - cc, kLandmarksFromTag, kNormalizedLandmarksToTag); - } - } else { - if (cc->Inputs().HasTag(kLandmarksToTag)) { - status = CopyVisibility( - cc, kNormalizedLandmarksFromTag, kLandmarksToTag); - } else { - status = CopyVisibility( - cc, kNormalizedLandmarksFromTag, kNormalizedLandmarksToTag); - } - } - - return status; -} - -template -absl::Status VisibilityCopyCalculator::CopyVisibility( - CalculatorContext* cc, const std::string& landmarks_from_tag, - const std::string& landmarks_to_tag) { - // Check that both landmarks to copy from and to copy to are non empty. - if (cc->Inputs().Tag(landmarks_from_tag).IsEmpty() || - cc->Inputs().Tag(landmarks_to_tag).IsEmpty()) { - return absl::OkStatus(); - } - - const auto landmarks_from = - cc->Inputs().Tag(landmarks_from_tag).Get(); - const auto landmarks_to = - cc->Inputs().Tag(landmarks_to_tag).Get(); - auto landmarks_out = absl::make_unique(); - - for (int i = 0; i < landmarks_from.landmark_size(); ++i) { - const auto& landmark_from = landmarks_from.landmark(i); - const auto& landmark_to = landmarks_to.landmark(i); - - // Create output landmark and copy all fields from the `to` landmark. - const auto& landmark_out = landmarks_out->add_landmark(); - *landmark_out = landmark_to; - - // Copy visibility and presence from the `from` landmark. - if (copy_visibility_) { - landmark_out->set_visibility(landmark_from.visibility()); - } - if (copy_presence_) { - landmark_out->set_presence(landmark_from.presence()); - } - } - - cc->Outputs() - .Tag(landmarks_to_tag) - .Add(landmarks_out.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/visibility_copy_calculator.proto b/mediapipe/calculators/util/visibility_copy_calculator.proto deleted file mode 100644 index df25937b8..000000000 --- a/mediapipe/calculators/util/visibility_copy_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator_options.proto"; - -message VisibilityCopyCalculatorOptions { - extend CalculatorOptions { - optional VisibilityCopyCalculatorOptions ext = 363728421; - } - - optional bool copy_visibility = 1 [default = true]; - - optional bool copy_presence = 2 [default = true]; -} diff --git a/mediapipe/calculators/util/visibility_smoothing_calculator.cc b/mediapipe/calculators/util/visibility_smoothing_calculator.cc deleted file mode 100644 index cd6ce5f0d..000000000 --- a/mediapipe/calculators/util/visibility_smoothing_calculator.cc +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/algorithm/container.h" -#include "mediapipe/calculators/util/visibility_smoothing_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/timestamp.h" -#include "mediapipe/util/filtering/low_pass_filter.h" - -namespace mediapipe { - -namespace { - -constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS"; -constexpr char kLandmarksTag[] = "LANDMARKS"; -constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS"; -constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS"; - -using mediapipe::LowPassFilter; - -// Abstract class for various visibility filters. -class VisibilityFilter { - public: - virtual ~VisibilityFilter() = default; - - virtual absl::Status Reset() { return absl::OkStatus(); } - - virtual absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - LandmarkList* out_landmarks) = 0; - - virtual absl::Status Apply(const NormalizedLandmarkList& in_landmarks, - const absl::Duration& timestamp, - NormalizedLandmarkList* out_landmarks) = 0; -}; - -// Returns visibility as is without smoothing. -class NoFilter : public VisibilityFilter { - public: - absl::Status Apply(const NormalizedLandmarkList& in_landmarks, - const absl::Duration& timestamp, - NormalizedLandmarkList* out_landmarks) override { - *out_landmarks = in_landmarks; - return absl::OkStatus(); - } - - absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - LandmarkList* out_landmarks) override { - *out_landmarks = in_landmarks; - return absl::OkStatus(); - } -}; - -// Please check LowPassFilter documentation for details. -class LowPassVisibilityFilter : public VisibilityFilter { - public: - LowPassVisibilityFilter(float alpha) : alpha_(alpha) {} - - absl::Status Reset() override { - visibility_filters_.clear(); - return absl::OkStatus(); - } - - absl::Status Apply(const LandmarkList& in_landmarks, - const absl::Duration& timestamp, - LandmarkList* out_landmarks) override { - return ApplyImpl(in_landmarks, timestamp, out_landmarks); - } - - absl::Status Apply(const NormalizedLandmarkList& in_landmarks, - const absl::Duration& timestamp, - NormalizedLandmarkList* out_landmarks) override { - return ApplyImpl(in_landmarks, timestamp, - out_landmarks); - } - - private: - template - absl::Status ApplyImpl(const LandmarksType& in_landmarks, - const absl::Duration& timestamp, - LandmarksType* out_landmarks) { - // Initializes filters for the first time or after Reset. If initialized - // then check the size. - int n_landmarks = in_landmarks.landmark_size(); - if (!visibility_filters_.empty()) { - RET_CHECK_EQ(visibility_filters_.size(), n_landmarks); - } else { - visibility_filters_.resize(n_landmarks, LowPassFilter(alpha_)); - } - - // Filter visibilities. - for (int i = 0; i < in_landmarks.landmark_size(); ++i) { - const auto& in_landmark = in_landmarks.landmark(i); - - auto* out_landmark = out_landmarks->add_landmark(); - *out_landmark = in_landmark; - out_landmark->set_visibility( - visibility_filters_[i].Apply(in_landmark.visibility())); - } - - return absl::OkStatus(); - } - - float alpha_; - std::vector visibility_filters_; -}; - -} // namespace - -// A calculator to smooth landmark visibilities over time. -// -// Exactly one landmarks input stream is expected. Output stream type should be -// the same as the input one. -// -// Inputs: -// LANDMARKS (optional): A LandmarkList of landmarks you want to smooth. -// NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want -// to smooth. -// -// Outputs: -// FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks. -// NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed -// landmarks. -// -// Example config: -// node { -// calculator: "VisibilitySmoothingCalculator" -// input_stream: "NORM_LANDMARKS:pose_landmarks" -// output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered" -// options: { -// [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { -// low_pass_filter: { -// alpha: 0.1 -// } -// } -// } -// } -// -class VisibilitySmoothingCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - std::unique_ptr visibility_filter_; -}; -REGISTER_CALCULATOR(VisibilitySmoothingCalculator); - -absl::Status VisibilitySmoothingCalculator::GetContract( - CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kNormalizedLandmarksTag) ^ - cc->Inputs().HasTag(kLandmarksTag)) - << "Exactly one landmarks input stream is expected"; - if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) { - cc->Inputs().Tag(kNormalizedLandmarksTag).Set(); - RET_CHECK(cc->Outputs().HasTag(kNormalizedFilteredLandmarksTag)) - << "Landmarks output stream should of the same type as input one"; - cc->Outputs() - .Tag(kNormalizedFilteredLandmarksTag) - .Set(); - } else { - cc->Inputs().Tag(kLandmarksTag).Set(); - RET_CHECK(cc->Outputs().HasTag(kFilteredLandmarksTag)) - << "Landmarks output stream should of the same type as input one"; - cc->Outputs().Tag(kFilteredLandmarksTag).Set(); - } - - return absl::OkStatus(); -} - -absl::Status VisibilitySmoothingCalculator::Open(CalculatorContext* cc) { - cc->SetOffset(TimestampDiff(0)); - - // Pick visibility filter. - const auto& options = cc->Options(); - if (options.has_no_filter()) { - visibility_filter_ = absl::make_unique(); - } else if (options.has_low_pass_filter()) { - visibility_filter_ = absl::make_unique( - options.low_pass_filter().alpha()); - } else { - RET_CHECK_FAIL() - << "Visibility filter is either not specified or not supported"; - } - - return absl::OkStatus(); -} - -absl::Status VisibilitySmoothingCalculator::Process(CalculatorContext* cc) { - // Check that landmarks are not empty and reset the filter if so. - // Don't emit an empty packet for this timestamp. - if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) && - cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) || - (cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().Tag(kLandmarksTag).IsEmpty())) { - MP_RETURN_IF_ERROR(visibility_filter_->Reset()); - return absl::OkStatus(); - } - - const auto& timestamp = - absl::Microseconds(cc->InputTimestamp().Microseconds()); - - if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) { - const auto& in_landmarks = - cc->Inputs().Tag(kNormalizedLandmarksTag).Get(); - auto out_landmarks = absl::make_unique(); - MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp, - out_landmarks.get())); - cc->Outputs() - .Tag(kNormalizedFilteredLandmarksTag) - .Add(out_landmarks.release(), cc->InputTimestamp()); - } else { - const auto& in_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - auto out_landmarks = absl::make_unique(); - MP_RETURN_IF_ERROR(visibility_filter_->Apply(in_landmarks, timestamp, - out_landmarks.get())); - cc->Outputs() - .Tag(kFilteredLandmarksTag) - .Add(out_landmarks.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/util/visibility_smoothing_calculator.proto b/mediapipe/calculators/util/visibility_smoothing_calculator.proto deleted file mode 100644 index 3b991923c..000000000 --- a/mediapipe/calculators/util/visibility_smoothing_calculator.proto +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator_options.proto"; - -message VisibilitySmoothingCalculatorOptions { - extend CalculatorOptions { - optional VisibilitySmoothingCalculatorOptions ext = 360207350; - } - - // Default behaviour and fast way to disable smoothing. - message NoFilter {} - - message LowPassFilter { - // Coefficient applied to a new value, whilte `1 - alpha` is applied to a - // stored value. Should be in [0, 1] range. The smaller the value - the - // smoother result and the bigger lag. - optional float alpha = 1 [default = 0.1]; - } - - oneof filter_options { - NoFilter no_filter = 1; - LowPassFilter low_pass_filter = 2; - } -} diff --git a/mediapipe/calculators/util/world_landmark_projection_calculator.cc b/mediapipe/calculators/util/world_landmark_projection_calculator.cc deleted file mode 100644 index bcd7352a2..000000000 --- a/mediapipe/calculators/util/world_landmark_projection_calculator.cc +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -namespace { - -constexpr char kLandmarksTag[] = "LANDMARKS"; -constexpr char kRectTag[] = "NORM_RECT"; - -} // namespace - -// Projects world landmarks from the rectangle to original coordinates. -// -// World landmarks are predicted in meters rather than in pixels of the image -// and have origin in the middle of the hips rather than in the corner of the -// pose image (cropped with given rectangle). Thus only rotation (but not scale -// and translation) is applied to the landmarks to transform them back to -// original coordinates. -// -// Input: -// LANDMARKS: A LandmarkList representing world landmarks in the rectangle. -// NORM_RECT: An NormalizedRect representing a normalized rectangle in image -// coordinates. (Optional) -// -// Output: -// LANDMARKS: A LandmarkList representing world landmarks projected (rotated -// but not scaled or translated) from the rectangle to original -// coordinates. -// -// Usage example: -// node { -// calculator: "WorldLandmarkProjectionCalculator" -// input_stream: "LANDMARKS:landmarks" -// input_stream: "NORM_RECT:rect" -// output_stream: "LANDMARKS:projected_landmarks" -// } -// -class WorldLandmarkProjectionCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kLandmarksTag).Set(); - if (cc->Inputs().HasTag(kRectTag)) { - cc->Inputs().Tag(kRectTag).Set(); - } - cc->Outputs().Tag(kLandmarksTag).Set(); - - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); - - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - // Check that landmarks and rect are not empty. - if (cc->Inputs().Tag(kLandmarksTag).IsEmpty() || - (cc->Inputs().HasTag(kRectTag) && - cc->Inputs().Tag(kRectTag).IsEmpty())) { - return absl::OkStatus(); - } - - const auto& in_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - std::function rotate_fn; - if (cc->Inputs().HasTag(kRectTag)) { - const auto& in_rect = cc->Inputs().Tag(kRectTag).Get(); - const float cosa = std::cos(in_rect.rotation()); - const float sina = std::sin(in_rect.rotation()); - rotate_fn = [cosa, sina](const Landmark& in_landmark, - Landmark* out_landmark) { - out_landmark->set_x(cosa * in_landmark.x() - sina * in_landmark.y()); - out_landmark->set_y(sina * in_landmark.x() + cosa * in_landmark.y()); - }; - } - - auto out_landmarks = absl::make_unique(); - for (int i = 0; i < in_landmarks.landmark_size(); ++i) { - const auto& in_landmark = in_landmarks.landmark(i); - - Landmark* out_landmark = out_landmarks->add_landmark(); - *out_landmark = in_landmark; - - if (rotate_fn) { - rotate_fn(in_landmark, out_landmark); - } - } - - cc->Outputs() - .Tag(kLandmarksTag) - .Add(out_landmarks.release(), cc->InputTimestamp()); - - return absl::OkStatus(); - } -}; -REGISTER_CALCULATOR(WorldLandmarkProjectionCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/BUILD b/mediapipe/calculators/video/BUILD deleted file mode 100644 index 806b9f1fa..000000000 --- a/mediapipe/calculators/video/BUILD +++ /dev/null @@ -1,555 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") -load( - "//mediapipe/framework/tool:mediapipe_graph.bzl", - "mediapipe_binary_graph", -) - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -proto_library( - name = "flow_to_image_calculator_proto", - srcs = ["flow_to_image_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "opencv_video_encoder_calculator_proto", - srcs = ["opencv_video_encoder_calculator.proto"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -proto_library( - name = "motion_analysis_calculator_proto", - srcs = ["motion_analysis_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/util/tracking:motion_analysis_proto", - ], -) - -proto_library( - name = "flow_packager_calculator_proto", - srcs = ["flow_packager_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/util/tracking:flow_packager_proto", - ], -) - -proto_library( - name = "box_tracker_calculator_proto", - srcs = ["box_tracker_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/util/tracking:box_tracker_proto", - ], -) - -proto_library( - name = "tracked_detection_manager_calculator_proto", - srcs = ["tracked_detection_manager_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/util/tracking:tracked_detection_manager_config_proto", - ], -) - -proto_library( - name = "box_detector_calculator_proto", - srcs = ["box_detector_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - "//mediapipe/util/tracking:box_detector_proto", - ], -) - -proto_library( - name = "video_pre_stream_calculator_proto", - srcs = ["video_pre_stream_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "motion_analysis_calculator_cc_proto", - srcs = ["motion_analysis_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util/tracking:motion_analysis_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":motion_analysis_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "flow_packager_calculator_cc_proto", - srcs = ["flow_packager_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util/tracking:flow_packager_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":flow_packager_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "box_tracker_calculator_cc_proto", - srcs = ["box_tracker_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util/tracking:box_tracker_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":box_tracker_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tracked_detection_manager_calculator_cc_proto", - srcs = ["tracked_detection_manager_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util/tracking:tracked_detection_manager_config_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":tracked_detection_manager_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "box_detector_calculator_cc_proto", - srcs = ["box_detector_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util/tracking:box_detector_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":box_detector_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "video_pre_stream_calculator_cc_proto", - srcs = ["video_pre_stream_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":video_pre_stream_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "flow_to_image_calculator_cc_proto", - srcs = ["flow_to_image_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":flow_to_image_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "opencv_video_encoder_calculator_cc_proto", - srcs = ["opencv_video_encoder_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":opencv_video_encoder_calculator_proto"], -) - -cc_library( - name = "flow_to_image_calculator", - srcs = ["flow_to_image_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":flow_to_image_calculator_cc_proto", - "//mediapipe/calculators/video/tool:flow_quantizer_model", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats/motion:optical_flow_field", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], - alwayslink = 1, -) - -cc_library( - name = "opencv_video_decoder_calculator", - srcs = ["opencv_video_decoder_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - ], - alwayslink = 1, -) - -cc_library( - name = "opencv_video_encoder_calculator", - srcs = ["opencv_video_encoder_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":opencv_video_encoder_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "tvl1_optical_flow_calculator", - srcs = ["tvl1_optical_flow_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats/motion:optical_flow_field", - "//mediapipe/framework/port:opencv_video", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/synchronization", - ], - alwayslink = 1, -) - -cc_library( - name = "motion_analysis_calculator", - srcs = ["motion_analysis_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":motion_analysis_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util/tracking:camera_motion", - "//mediapipe/util/tracking:camera_motion_cc_proto", - "//mediapipe/util/tracking:frame_selection_cc_proto", - "//mediapipe/util/tracking:motion_analysis", - "//mediapipe/util/tracking:motion_estimation", - "//mediapipe/util/tracking:motion_models", - "//mediapipe/util/tracking:region_flow_cc_proto", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "flow_packager_calculator", - srcs = ["flow_packager_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":flow_packager_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/util/tracking:camera_motion_cc_proto", - "//mediapipe/util/tracking:flow_packager", - "//mediapipe/util/tracking:region_flow_cc_proto", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], - alwayslink = 1, -) - -cc_library( - name = "box_tracker_calculator", - srcs = ["box_tracker_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":box_tracker_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", # fixdeps: keep -- required for exobazel build. - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:options_util", - "//mediapipe/util/tracking", - "//mediapipe/util/tracking:box_tracker", - "//mediapipe/util/tracking:tracking_visualization_utilities", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/container:node_hash_set", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "box_detector_calculator", - srcs = ["box_detector_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":box_detector_calculator_cc_proto", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", # fixdeps: keep -- required for exobazel build. - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_features2d", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/util:resource_util", - "//mediapipe/util/tracking", - "//mediapipe/util/tracking:box_detector", - "//mediapipe/util/tracking:box_tracker", - "//mediapipe/util/tracking:box_tracker_cc_proto", - "//mediapipe/util/tracking:flow_packager_cc_proto", - "//mediapipe/util/tracking:tracking_visualization_utilities", - ] + select({ - "//mediapipe:android": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:ios": [ - "//mediapipe/util/android/file/base", - ], - "//mediapipe:macos": [ - "//mediapipe/framework/port:file_helpers", - ], - "//conditions:default": [ - "//mediapipe/framework/port:file_helpers", - ], - }), - alwayslink = 1, -) - -cc_library( - name = "tracked_detection_manager_calculator", - srcs = ["tracked_detection_manager_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tracked_detection_manager_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:status", - "//mediapipe/util/tracking", - "//mediapipe/util/tracking:box_tracker", - "//mediapipe/util/tracking:tracked_detection", - "//mediapipe/util/tracking:tracked_detection_manager", - "//mediapipe/util/tracking:tracking_visualization_utilities", - "@com_google_absl//absl/container:node_hash_map", - ], - alwayslink = 1, -) - -cc_library( - name = "video_pre_stream_calculator", - srcs = ["video_pre_stream_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":video_pre_stream_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:video_stream_header", - ], - alwayslink = 1, -) - -filegroup( - name = "test_videos", - srcs = [ - "testdata/format_FLV_H264_AAC.video", - "testdata/format_MKV_VP8_VORBIS.video", - "testdata/format_MP4_AVC720P_AAC.video", - ], - visibility = ["//visibility:public"], -) - -cc_test( - name = "opencv_video_decoder_calculator_test", - srcs = ["opencv_video_decoder_calculator_test.cc"], - data = [":test_videos"], - deps = [ - ":opencv_video_decoder_calculator", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/flags:flag", - ], -) - -cc_test( - name = "opencv_video_encoder_calculator_test", - srcs = ["opencv_video_encoder_calculator_test.cc"], - data = [":test_videos"], - deps = [ - ":opencv_video_decoder_calculator", - ":opencv_video_encoder_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:deleting_file", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/flags:flag", - ], -) - -cc_test( - name = "tvl1_optical_flow_calculator_test", - srcs = ["tvl1_optical_flow_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":tvl1_optical_flow_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats/motion:optical_flow_field", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -mediapipe_binary_graph( - name = "parallel_tracker_binarypb", - graph = "testdata/parallel_tracker_graph.pbtxt", - output_name = "testdata/parallel_tracker.binarypb", - visibility = ["//visibility:public"], - deps = [ - ":box_tracker_calculator", - ":flow_packager_calculator", - ":motion_analysis_calculator", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", - ], -) - -mediapipe_binary_graph( - name = "tracker_binarypb", - graph = "testdata/tracker_graph.pbtxt", - output_name = "testdata/tracker.binarypb", - visibility = ["//visibility:public"], - deps = [ - ":box_tracker_calculator", - ":flow_packager_calculator", - ":motion_analysis_calculator", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", - ], -) - -cc_test( - name = "tracking_graph_test", - size = "small", - srcs = ["tracking_graph_test.cc"], - copts = ["-DPARALLEL_INVOKER_ACTIVE"] + select({ - "//mediapipe:apple": [], - "//mediapipe:android": [], - "//conditions:default": [], - }), - data = [ - ":testdata/lenna.png", - ":testdata/parallel_tracker.binarypb", - ":testdata/tracker.binarypb", - ], - deps = [ - ":box_tracker_calculator", - ":box_tracker_calculator_cc_proto", - ":flow_packager_calculator", - ":motion_analysis_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:packet", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:advanced_proto", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", - "//mediapipe/util/tracking:box_tracker_cc_proto", - "//mediapipe/util/tracking:tracking_cc_proto", - "@com_google_absl//absl/flags:flag", - ], -) - -cc_test( - name = "video_pre_stream_calculator_test", - srcs = ["video_pre_stream_calculator_test.cc"], - deps = [ - ":video_pre_stream_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) diff --git a/mediapipe/calculators/video/box_detector_calculator.cc b/mediapipe/calculators/video/box_detector_calculator.cc deleted file mode 100644 index b7b91d253..000000000 --- a/mediapipe/calculators/video/box_detector_calculator.cc +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - -#include "absl/memory/memory.h" -#include "absl/strings/numbers.h" -#include "mediapipe/calculators/video/box_detector_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_features2d_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/resource_util.h" -#include "mediapipe/util/tracking/box_detector.h" -#include "mediapipe/util/tracking/box_tracker.h" -#include "mediapipe/util/tracking/box_tracker.pb.h" -#include "mediapipe/util/tracking/flow_packager.pb.h" -#include "mediapipe/util/tracking/tracking.h" -#include "mediapipe/util/tracking/tracking_visualization_utilities.h" - -#if defined(MEDIAPIPE_MOBILE) -#include "mediapipe/util/android/file/base/file.h" -#include "mediapipe/util/android/file/base/helpers.h" -#else -#include "mediapipe/framework/port/file_helpers.h" -#endif - -namespace mediapipe { - -// A calculator to detect reappeared box positions from single frame. -// -// Input stream: -// TRACKING: Input tracking data (proto TrackingData) containing features and -// descriptors. -// VIDEO: Optional input video stream tracked boxes are rendered over -// (Required if VIZ is specified). -// FEATURES: Input feature points (std::vector) in the original -// pixel space. -// DESCRIPTORS: Input feature descriptors (std::vector). Actual feature -// dimension needs to be specified in detector_options. -// IMAGE_SIZE: Input image dimension. -// TRACKED_BOXES : input box tracking result (proto TimedBoxProtoList) from -// BoxTrackerCalculator. -// ADD_INDEX: Optional std::string containing binary format proto of type -// BoxDetectorIndex. Used for adding target index to the detector -// search index during runtime. -// CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended -// to be used with SyncSetInputStreamHandler. -// REACQ_SWITCH: Optional bool for swithcing on and off reacquisition -// functionality. User should initialize a graph with box detector -// calculator and be able to switch it on and off in runtime. -// -// Output streams: -// VIZ: Optional output video stream with rendered box positions -// (requires VIDEO to be present) -// BOXES: Optional output stream of type TimedBoxProtoList for each lost box. -// -// Imput side packets: -// INDEX_PROTO_STRING: Optional std::string containing binary format proto of -// type -// BoxDetectorIndex. Used for initializing box_detector -// with predefined template images. -// FRAME_ALIGNMENT: Optional integer to indicate alignment_boundary for -// outputing ImageFrame in "VIZ" stream. -// Set to ImageFrame::kDefaultAlignmentBoundary for -// offline pipeline to be compatible with FFmpeg. -// Set to ImageFrame::kGlDefaultAlignmentBoundary for Apps -// to be compatible with GL renderer. -// OUTPUT_INDEX_FILENAME: File path to the output index file. - -class BoxDetectorCalculator : public CalculatorBase { - public: - ~BoxDetectorCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - BoxDetectorCalculatorOptions options_; - std::unique_ptr box_detector_; - bool detector_switch_ = true; - uint32 frame_alignment_ = ImageFrame::kDefaultAlignmentBoundary; - bool write_index_ = false; - int box_id_ = 0; -}; - -REGISTER_CALCULATOR(BoxDetectorCalculator); - -absl::Status BoxDetectorCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag("TRACKING")) { - cc->Inputs().Tag("TRACKING").Set(); - } - - if (cc->Inputs().HasTag("TRACKED_BOXES")) { - cc->Inputs().Tag("TRACKED_BOXES").Set(); - } - - if (cc->Inputs().HasTag("VIDEO")) { - cc->Inputs().Tag("VIDEO").Set(); - } - - if (cc->Inputs().HasTag("FEATURES")) { - RET_CHECK(cc->Inputs().HasTag("DESCRIPTORS")) - << "FEATURES and DESCRIPTORS need to be specified together."; - cc->Inputs().Tag("FEATURES").Set>(); - } - - if (cc->Inputs().HasTag("DESCRIPTORS")) { - RET_CHECK(cc->Inputs().HasTag("FEATURES")) - << "FEATURES and DESCRIPTORS need to be specified together."; - cc->Inputs().Tag("DESCRIPTORS").Set>(); - } - - if (cc->Inputs().HasTag("IMAGE_SIZE")) { - cc->Inputs().Tag("IMAGE_SIZE").Set>(); - } - - if (cc->Inputs().HasTag("ADD_INDEX")) { - cc->Inputs().Tag("ADD_INDEX").Set(); - } - - if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) { - cc->Inputs().Tag("CANCEL_OBJECT_ID").Set(); - } - - if (cc->Inputs().HasTag("REACQ_SWITCH")) { - cc->Inputs().Tag("REACQ_SWITCH").Set(); - } - - if (cc->Outputs().HasTag("BOXES")) { - cc->Outputs().Tag("BOXES").Set(); - } - - if (cc->Outputs().HasTag("VIZ")) { - RET_CHECK(cc->Inputs().HasTag("VIDEO")) - << "Output stream VIZ requires VIDEO to be present."; - cc->Outputs().Tag("VIZ").Set(); - } - - if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) { - cc->InputSidePackets().Tag("INDEX_PROTO_STRING").Set(); - } - - if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) { - cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Set(); - } - - if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) { - cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Set(); - } - - return absl::OkStatus(); -} - -absl::Status BoxDetectorCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - box_detector_ = BoxDetectorInterface::Create(options_.detector_options()); - - if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) { - BoxDetectorIndex predefined_index; - if (!predefined_index.ParseFromString(cc->InputSidePackets() - .Tag("INDEX_PROTO_STRING") - .Get())) { - LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING"; - } - box_detector_->AddBoxDetectorIndex(predefined_index); - } - - for (const auto& filename : options_.index_proto_filename()) { - std::string string_path; - ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(filename)); - std::string index_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string)); - BoxDetectorIndex predefined_index; - if (!predefined_index.ParseFromString(index_string)) { - LOG(FATAL) - << "failed to parse BoxDetectorIndex from index_proto_filename"; - } - box_detector_->AddBoxDetectorIndex(predefined_index); - } - - if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) { - write_index_ = true; - } - - if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) { - frame_alignment_ = cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Get(); - } - - return absl::OkStatus(); -} - -absl::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { - const Timestamp timestamp = cc->InputTimestamp(); - const int64 timestamp_msec = timestamp.Value() / 1000; - - InputStream* cancel_object_id_stream = - cc->Inputs().HasTag("CANCEL_OBJECT_ID") - ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID")) - : nullptr; - if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) { - const int cancel_object_id = cancel_object_id_stream->Get(); - box_detector_->CancelBoxDetection(cancel_object_id); - } - - InputStream* add_index_stream = cc->Inputs().HasTag("ADD_INDEX") - ? &(cc->Inputs().Tag("ADD_INDEX")) - : nullptr; - if (add_index_stream && !add_index_stream->IsEmpty()) { - BoxDetectorIndex predefined_index; - if (!predefined_index.ParseFromString( - add_index_stream->Get())) { - LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX"; - } - box_detector_->AddBoxDetectorIndex(predefined_index); - } - - InputStream* reacq_switch_stream = cc->Inputs().HasTag("REACQ_SWITCH") - ? &(cc->Inputs().Tag("REACQ_SWITCH")) - : nullptr; - if (reacq_switch_stream && !reacq_switch_stream->IsEmpty()) { - detector_switch_ = reacq_switch_stream->Get(); - } - - if (!detector_switch_) { - return absl::OkStatus(); - } - - InputStream* track_stream = cc->Inputs().HasTag("TRACKING") - ? &(cc->Inputs().Tag("TRACKING")) - : nullptr; - InputStream* video_stream = - cc->Inputs().HasTag("VIDEO") ? &(cc->Inputs().Tag("VIDEO")) : nullptr; - InputStream* feature_stream = cc->Inputs().HasTag("FEATURES") - ? &(cc->Inputs().Tag("FEATURES")) - : nullptr; - InputStream* descriptor_stream = cc->Inputs().HasTag("DESCRIPTORS") - ? &(cc->Inputs().Tag("DESCRIPTORS")) - : nullptr; - - CHECK(track_stream != nullptr || video_stream != nullptr || - (feature_stream != nullptr && descriptor_stream != nullptr)) - << "One and only one of {tracking_data, input image frame, " - "feature/descriptor} need to be valid."; - - InputStream* tracked_boxes_stream = cc->Inputs().HasTag("TRACKED_BOXES") - ? &(cc->Inputs().Tag("TRACKED_BOXES")) - : nullptr; - std::unique_ptr detected_boxes(new TimedBoxProtoList()); - - if (track_stream != nullptr) { - // Detect from tracking data - if (track_stream->IsEmpty()) { - return absl::OkStatus(); - } - - const TrackingData& tracking_data = track_stream->Get(); - - CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed."; - - const TimedBoxProtoList tracked_boxes = - tracked_boxes_stream->Get(); - - box_detector_->DetectAndAddBox(tracking_data, tracked_boxes, timestamp_msec, - detected_boxes.get()); - } else if (video_stream != nullptr) { - // Detect from input frame - if (video_stream->IsEmpty()) { - return absl::OkStatus(); - } - - TimedBoxProtoList tracked_boxes; - if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) { - tracked_boxes = tracked_boxes_stream->Get(); - } - - // Just directly pass along the image frame data as-is for detection; we - // don't need to worry about conforming to a specific alignment here. - const cv::Mat input_view = - formats::MatView(&video_stream->Get()); - box_detector_->DetectAndAddBox(input_view, tracked_boxes, timestamp_msec, - detected_boxes.get()); - } else { - if (feature_stream->IsEmpty() || descriptor_stream->IsEmpty()) { - return absl::OkStatus(); - } - - const auto& image_size = - cc->Inputs().Tag("IMAGE_SIZE").Get>(); - float inv_scale = 1.0f / std::max(image_size.first, image_size.second); - - TimedBoxProtoList tracked_boxes; - if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) { - tracked_boxes = tracked_boxes_stream->Get(); - } else if (write_index_) { - auto* box_ptr = tracked_boxes.add_box(); - box_ptr->set_id(box_id_); - box_ptr->set_reacquisition(true); - box_ptr->set_aspect_ratio((float)image_size.first / - (float)image_size.second); - - box_ptr->mutable_quad()->add_vertices(0); - box_ptr->mutable_quad()->add_vertices(0); - - box_ptr->mutable_quad()->add_vertices(0); - box_ptr->mutable_quad()->add_vertices(1); - - box_ptr->mutable_quad()->add_vertices(1); - box_ptr->mutable_quad()->add_vertices(1); - - box_ptr->mutable_quad()->add_vertices(1); - box_ptr->mutable_quad()->add_vertices(0); - - ++box_id_; - } - - const auto& features = feature_stream->Get>(); - const int feature_size = features.size(); - std::vector features_vec(feature_size); - - const auto& descriptors = descriptor_stream->Get>(); - const int dims = options_.detector_options().descriptor_dims(); - CHECK_GE(descriptors.size(), feature_size * dims); - cv::Mat descriptors_mat(feature_size, dims, CV_32F); - for (int j = 0; j < feature_size; ++j) { - features_vec[j].Set(features[j].pt.x * inv_scale, - features[j].pt.y * inv_scale); - for (int i = 0; i < dims; ++i) { - descriptors_mat.at(j, i) = descriptors[j * dims + i]; - } - } - - box_detector_->DetectAndAddBoxFromFeatures( - features_vec, descriptors_mat, tracked_boxes, timestamp_msec, - image_size.first * inv_scale, image_size.second * inv_scale, - detected_boxes.get()); - } - - if (cc->Outputs().HasTag("VIZ")) { - cv::Mat viz_view; - std::unique_ptr viz_frame; - if (video_stream != nullptr && !video_stream->IsEmpty()) { - viz_frame = absl::make_unique(); - viz_frame->CopyFrom(video_stream->Get(), frame_alignment_); - viz_view = formats::MatView(viz_frame.get()); - } - for (const auto& box : detected_boxes->box()) { - RenderBox(box, &viz_view); - } - cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp); - } - - if (cc->Outputs().HasTag("BOXES")) { - cc->Outputs().Tag("BOXES").Add(detected_boxes.release(), timestamp); - } - - return absl::OkStatus(); -} - -absl::Status BoxDetectorCalculator::Close(CalculatorContext* cc) { - if (write_index_) { - BoxDetectorIndex index = box_detector_->ObtainBoxDetectorIndex(); - MEDIAPIPE_CHECK_OK(mediapipe::file::SetContents( - cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Get(), - index.SerializeAsString())); - } - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/box_detector_calculator.proto b/mediapipe/calculators/video/box_detector_calculator.proto deleted file mode 100644 index f0eb42f16..000000000 --- a/mediapipe/calculators/video/box_detector_calculator.proto +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/tracking/box_detector.proto"; - -message BoxDetectorCalculatorOptions { - extend CalculatorOptions { - optional BoxDetectorCalculatorOptions ext = 289746530; - } - - optional BoxDetectorOptions detector_options = 1; - - // File path to the template index files. - repeated string index_proto_filename = 2; -} diff --git a/mediapipe/calculators/video/box_tracker_calculator.cc b/mediapipe/calculators/video/box_tracker_calculator.cc deleted file mode 100644 index 7d04d9765..000000000 --- a/mediapipe/calculators/video/box_tracker_calculator.cc +++ /dev/null @@ -1,1286 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include - -#include "absl/container/flat_hash_set.h" -#include "absl/container/node_hash_map.h" -#include "absl/container/node_hash_set.h" -#include "absl/strings/numbers.h" -#include "mediapipe/calculators/video/box_tracker_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/options_util.h" -#include "mediapipe/util/tracking/box_tracker.h" -#include "mediapipe/util/tracking/tracking.h" -#include "mediapipe/util/tracking/tracking_visualization_utilities.h" - -namespace mediapipe { - -const char kOptionsTag[] = "OPTIONS"; - -// A calculator to track box positions over time. -// This calculator works in two modes: -// a) Streaming mode, forward tracking only uses per frame TRACKING TrackingData -// supplied by tracking. For faster processing use TRACK_TIME to explicitly -// request tracking results at higher FPS than supplied by TRACKING. -// b) Batch mode: Tracks from tracking chunk files as specified by CACHE_DIR -// side packet (forward and backward with multiple key framing support). -// NOTE: When using batch mode you might need some external logic -// to clear out the caching directory between runs / files; or otherwise -// stale chunk files might be used. -// -// Initial positions can be either supplied via calculator options or -// INITIAL_POS (not supported on mobile) side packet, but not both. - -// Input stream: -// TRACKING: Input tracking data (proto TrackingData, required if CACHE_DIR -// is not defined) -// TRACK_TIME: Timestamps that tracking results should be generated at. -// Optional. Results generated at a TRACK_TIME w/o corresponding -// TRACKING packet will be queued up and returned when the next -// TRACKING input is observed. For those packets also no -// visualization output will be generated. -// Can be Packet of any type. -// VIDEO: Optional input video stream tracked boxes are rendered over. -// START: Optional input stream with PreStream packet to begin processing. -// Typical use case: When used in batch mode have -// FlowPackagerCalculator emit a COMPLETE packet to indicate caching -// is completed. -// START_POS: Optional initial positions to be tracked as TimedBoxProtoList. -// Timestamp of the box is used, so box timestamp does not have to -// be monotonic. Assign monotonic increasing timestamps for -// START_POS, e.g. 1,2,3 per request. -// Supplied starting positions are 'fast forwarded', i.e. quickly -// tracked towards current track head, i.e. last received -// TrackingData and added to current set of tracked boxes. -// This is recommended to be used with SyncSetInputStreamHandler. -// START_POS_PROTO_STRING: Same as START_POS, but is in the form of serialized -// protobuffer std::string. When both START_POS and -// START_POS_PROTO_STRING are present, START_POS is used. Suggest -// to specify only one of them. -// RESTART_POS: Same as START_POS, but exclusively for receiving detection -// results from reacquisition. -// CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended -// to be used with SyncSetInputStreamHandler. -// RA_TRACK: Performs random access tracking within the specified -// tracking cache, which is specified in the options of this -// calculator BoxTrackerCalculatorOptions. Input is of type -// TimedBoxProtoList. -// Assumed to be supplied as pair -// [start0, stop0, start1, stop1, ...] of boxes, -// (that is list size() % 2 == 0), where position, id and time -// is used for start, and only time for stop; that is position -// is ignored. -// Assign monotonically increasing packet timestamps for RA_TRACK, -// e.g. 1,2,3; however the timestamp in TimedBoxProtoList -// can be in arbitrary order. -// Use with SyncSetInputStreamHandler in streaming mode only. -// RA_TRACK_PROTO_STRING: Same as RA_TRACK, but is in the form of serialized -// protobuffer std::string. When both RA_TRACK and -// RA_TRACK_PROTO_STRING are present, RA_TRACK is used. Suggest -// to specify only one of them. -// -// Output streams: -// VIZ: Optional output video stream with rendered box positions -// (requires VIDEO to be present) -// BOXES: Optional output stream of type TimedBoxProtoList for each -// initialized result. -// RA_BOXES: Optional output stream of type TimedBoxProtoList for each -// request in RA_TRACK. Same timestamp as request is used. -// -// Input side packets: -// INITIAL_POS: Optional initial positions to be tracked as text format proto -// of type TimedBoxProtoList. Can not be combined with initial -// position option. NOT SUPPORTED ON MOBILE. -// CACHE_DIR: Optional caching directory tracking chunk files are read -// from. -// -// -class BoxTrackerCalculator : public CalculatorBase { - public: - ~BoxTrackerCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - protected: - void RenderStates(const std::vector& states, cv::Mat* mat); - void RenderInternalStates(const std::vector& states, - cv::Mat* mat); - - // MotionBox and corresponding PathSegment of results; used in streaming mode. - struct MotionBoxPath { - MotionBoxPath(MotionBox&& box_, PathSegment&& path_, bool reacq_ = false) - : box(std::move(box_)), path(std::move(path_)), reacquisition(reacq_) {} - - MotionBoxPath() = default; - - // Trims the state for forward/backward tracking. - void Trim(const int cache_size, const bool forward) { - if (forward) { - // Trims the box's states queue. - box.TrimFront(cache_size); - // Trims the path queue. - int trim_count = path.size() - cache_size; - while (trim_count-- > 0) { - path.pop_front(); - } - } else { // backward - // Trims the box's states queue. - box.TrimBack(cache_size); - // Trims the path queue. - int trim_count = path.size() - cache_size; - while (trim_count-- > 0) { - path.pop_back(); - } - } - } - - MotionBox box; - PathSegment path; - bool reacquisition; - }; - - // MotionBoxPath per unique id that we are tracking. - typedef absl::node_hash_map MotionBoxMap; - - // Performs tracking of all MotionBoxes in box_map by one frame forward or - // backward to or from data_frame_num using passed TrackingData. - // Specify destination timestamp and frame duration TrackingData was - // computed for. Used in streaming mode. - // Returns list of ids that failed. - void StreamTrack(const TrackingData& data, int data_frame_num, - int64 dst_timestamp_ms, int64 duration_ms, bool forward, - MotionBoxMap* box_map, std::vector* failed_ids); - - // Fast forwards specified boxes from starting position to current play head - // and outputs successful boxes to box_map. - // Specify the timestamp boxes are tracked from via timestamp in each - // TimedBox. - void FastForwardStartPos(const TimedBoxProtoList& start_pos_list, - MotionBoxMap* box_map); - - // Performs random access tracking from box_list (start,stop) tuples and - // outputs results. - void OutputRandomAccessTrack(const TimedBoxProtoList& box_list, - CalculatorContext* cc); - - private: - BoxTrackerCalculatorOptions options_; - - TimedBoxProtoList initial_pos_; - - // Keeps tracks boxes that have already been initialized. - absl::node_hash_set initialized_ids_; - - // Non empty for batch mode tracking. - std::string cache_dir_; - // Ids to be tracked in batch_mode. - absl::node_hash_set batch_track_ids_; - - int frame_num_ = 0; - - // Boxes that are tracked in streaming mode. - MotionBoxMap streaming_motion_boxes_; - - absl::node_hash_map> last_tracked_boxes_; - int frame_num_since_reset_ = 0; - - // Cache used during streaming mode for fast forward tracking. - std::deque> tracking_data_cache_; - - // Indicator to track if box_tracker_ has started tracking. - bool tracking_issued_ = false; - std::unique_ptr box_tracker_; - - // If set, renders tracking data into VIZ stream. - bool visualize_tracking_data_ = false; - - // If set, renders the box state and internal box state into VIZ stream. - bool visualize_state_ = false; - bool visualize_internal_state_ = false; - - // Timestamps for every tracking data input frame. - std::deque track_timestamps_; - - // For pruning track_timestamps_ queue. - static const int kTrackTimestampsMinQueueSize; - - // The long-running index of the head of track_timestamps_. - int track_timestamps_base_index_ = 0; - - // For pruning MotionBoxPath's state/path queues. - static const int kMotionBoxPathMinQueueSize; - - // Queued track time requests. - std::vector queued_track_requests_; - - // Stores the tracked ids that have been discarded actively, from continuous - // tracking data. It may accumulate across multiple frames. Once consumed, it - // should be cleared immediately. - absl::flat_hash_set actively_discarded_tracked_ids_; - - // Add smooth transition between re-acquisition and previous tracked boxes. - // `result_box` is the tracking result of one specific timestamp. The smoothed - // result will be updated in place. - // `subframe_alpha` is from 0 to 1 (0, 1 repressents previous and current - // frame with TRACKING_DATA). Any frames with TRACK_TIME should interpolate in - // between. - void AddSmoothTransitionToOutputBox(int box_id, TimedBox* result_box, - float subframe_alpha = 1.0f); - - std::deque::iterator GetRandomAccessTimestampPos( - const TimedBoxProto& start, bool forward_track); - - std::deque>::iterator - GetRandomAccessStartData( - const std::deque::iterator& timestamp_pos); - - MotionBoxMap PrepareRandomAccessTrack( - const TimedBoxProto& start, int init_frame, bool forward_track, - const std::deque>::iterator& - start_data); - - bool RunForwardTrack( - const std::deque>::iterator& - start_data, - int init_frame, MotionBoxMap* single_map, int64 end_time_msec); - - bool RunBackwardTrack( - const std::deque>::iterator& - start_data, - int init_frame, MotionBoxMap* single_map, int64 end_time_msec); - - void ObtainResultOfRandomAccessTrack( - const MotionBoxMap& single_map, const TimedBoxProto& start, - int64 end_time_msec, - const std::unique_ptr& result_list); -}; - -REGISTER_CALCULATOR(BoxTrackerCalculator); - -// At least 2 timestamps need to be present in track_timestamps_ or streaming -// logic's duration calculation will break. -const int BoxTrackerCalculator::kTrackTimestampsMinQueueSize = 2; - -// At least 2: the newly added state, and one from the history. -const int BoxTrackerCalculator::kMotionBoxPathMinQueueSize = 2; - -namespace { - -// Convert box position according to rotation angle in degrees. -void ConvertCoordinateForRotation(float in_top, float in_left, float in_bottom, - float in_right, int rotation, float* out_top, - float* out_left, float* out_bottom, - float* out_right) { - CHECK(out_top != nullptr); - CHECK(out_left != nullptr); - CHECK(out_bottom != nullptr); - CHECK(out_right != nullptr); - const float in_center_x = (in_left + in_right) * 0.5f; - const float in_center_y = (in_top + in_bottom) * 0.5f; - const float in_width = in_right - in_left; - const float in_height = in_bottom - in_top; - CHECK_GT(in_width, 0); - CHECK_GT(in_height, 0); - float out_center_x; - float out_center_y; - float out_width; - float out_height; - switch (rotation) { - case 0: - out_center_x = in_center_x; - out_center_y = in_center_y; - out_width = in_width; - out_height = in_height; - break; - case -270: // FALL_THROUGH_INTENDED - case 90: - out_center_x = 1 - in_center_y; - out_center_y = in_center_x; - out_width = in_height; - out_height = in_width; - break; - case -180: // FALL_THROUGH_INTENDED - case 180: - out_center_x = 1 - in_center_x; - out_center_y = 1 - in_center_y; - out_width = in_width; - out_height = in_height; - break; - case -90: // FALL_THROUGH_INTENDED - case 270: - out_center_x = in_center_y; - out_center_y = 1 - in_center_x; - out_width = in_height; - out_height = in_width; - break; - default: - LOG(ERROR) << "invalid rotation " << rotation; - out_center_x = in_center_x; - out_center_y = in_center_y; - out_width = in_width; - out_height = in_height; - break; - } - *out_top = out_center_y - out_height * 0.5f; - *out_left = out_center_x - out_width * 0.5f; - *out_bottom = out_center_y + out_height * 0.5f; - *out_right = out_center_x + out_width * 0.5f; -} - -void AddStateToPath(const MotionBoxState& state, int64 time_msec, - PathSegment* path) { - CHECK(path); - TimedBox result; - TimedBoxFromMotionBoxState(state, &result); - result.time_msec = time_msec; - - const auto insert_pos = std::lower_bound(path->begin(), path->end(), result); - // Do not duplicate box positions. - if (insert_pos == path->end() || insert_pos->time_msec != time_msec) { - path->insert(insert_pos, - InternalTimedBox(result, new MotionBoxState(state))); - } else { - LOG(ERROR) << "Box at time " << time_msec << " already present; ignoring"; - } -} - -} // namespace. - -absl::Status BoxTrackerCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag("TRACKING")) { - cc->Inputs().Tag("TRACKING").Set(); - } - - if (cc->Inputs().HasTag("TRACK_TIME")) { - RET_CHECK(cc->Inputs().HasTag("TRACKING")) - << "TRACK_TIME needs TRACKING input"; - cc->Inputs().Tag("TRACK_TIME").SetAny(); - } - - if (cc->Inputs().HasTag("VIDEO")) { - cc->Inputs().Tag("VIDEO").Set(); - } - - if (cc->Inputs().HasTag("START")) { - // Actual packet content does not matter. - cc->Inputs().Tag("START").SetAny(); - } - - if (cc->Inputs().HasTag("START_POS")) { - cc->Inputs().Tag("START_POS").Set(); - } - - if (cc->Inputs().HasTag("START_POS_PROTO_STRING")) { - cc->Inputs().Tag("START_POS_PROTO_STRING").Set(); - } - - if (cc->Inputs().HasTag("RESTART_POS")) { - cc->Inputs().Tag("RESTART_POS").Set(); - } - - if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) { - cc->Inputs().Tag("CANCEL_OBJECT_ID").Set(); - } - - if (cc->Inputs().HasTag("RA_TRACK")) { - cc->Inputs().Tag("RA_TRACK").Set(); - } - - if (cc->Inputs().HasTag("RA_TRACK_PROTO_STRING")) { - cc->Inputs().Tag("RA_TRACK_PROTO_STRING").Set(); - } - - if (cc->Outputs().HasTag("VIZ")) { - RET_CHECK(cc->Inputs().HasTag("VIDEO")) - << "Output stream VIZ requires VIDEO to be present."; - cc->Outputs().Tag("VIZ").Set(); - } - - if (cc->Outputs().HasTag("BOXES")) { - cc->Outputs().Tag("BOXES").Set(); - } - - if (cc->Outputs().HasTag("RA_BOXES")) { - cc->Outputs().Tag("RA_BOXES").Set(); - } - -#if defined(__ANDROID__) || defined(__APPLE__) || defined(__EMSCRIPTEN__) - RET_CHECK(!cc->InputSidePackets().HasTag("INITIAL_POS")) - << "Unsupported on mobile"; -#else - if (cc->InputSidePackets().HasTag("INITIAL_POS")) { - cc->InputSidePackets().Tag("INITIAL_POS").Set(); - } -#endif // defined(__ANDROID__) || defined(__APPLE__) || defined(__EMSCRIPTEN__) - - if (cc->InputSidePackets().HasTag("CACHE_DIR")) { - cc->InputSidePackets().Tag("CACHE_DIR").Set(); - } - - RET_CHECK(cc->Inputs().HasTag("TRACKING") != - cc->InputSidePackets().HasTag("CACHE_DIR")) - << "Either TRACKING or CACHE_DIR needs to be specified."; - - if (cc->InputSidePackets().HasTag(kOptionsTag)) { - cc->InputSidePackets().Tag(kOptionsTag).Set(); - } - - return absl::OkStatus(); -} - -absl::Status BoxTrackerCalculator::Open(CalculatorContext* cc) { - options_ = tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), kOptionsTag); - - RET_CHECK(!cc->InputSidePackets().HasTag("INITIAL_POS") || - !options_.has_initial_position()) - << "Can not specify initial position as side packet and via options"; - - if (options_.has_initial_position()) { - initial_pos_ = options_.initial_position(); - } - -#if !defined(__ANDROID__) && !defined(__APPLE__) && !defined(__EMSCRIPTEN__) - if (cc->InputSidePackets().HasTag("INITIAL_POS")) { - LOG(INFO) << "Parsing: " - << cc->InputSidePackets().Tag("INITIAL_POS").Get(); - initial_pos_ = ParseTextProtoOrDie( - cc->InputSidePackets().Tag("INITIAL_POS").Get()); - } -#endif // !defined(__ANDROID__) && !defined(__APPLE__) && - // !defined(__EMSCRIPTEN__) - - // Compile list of ids to be tracked. - for (const auto& pos : initial_pos_.box()) { - RET_CHECK(pos.id() >= 0) << "Requires id to be set"; - batch_track_ids_.insert(pos.id()); - } - - visualize_tracking_data_ = - options_.visualize_tracking_data() && cc->Outputs().HasTag("VIZ"); - visualize_state_ = options_.visualize_state() && cc->Outputs().HasTag("VIZ"); - visualize_internal_state_ = - options_.visualize_internal_state() && cc->Outputs().HasTag("VIZ"); - - // Force recording of internal state for rendering. - if (visualize_internal_state_) { - options_.mutable_tracker_options() - ->mutable_track_step_options() - ->set_return_internal_state(true); - } - - if (visualize_state_ || visualize_internal_state_) { - options_.mutable_tracker_options()->set_record_path_states(true); - } - - if (cc->InputSidePackets().HasTag("CACHE_DIR")) { - cache_dir_ = cc->InputSidePackets().Tag("CACHE_DIR").Get(); - RET_CHECK(!cache_dir_.empty()); - box_tracker_.reset(new BoxTracker(cache_dir_, options_.tracker_options())); - } else { - // Check that all boxes have a unique id. - RET_CHECK(initial_pos_.box_size() == batch_track_ids_.size()) - << "In streaming mode every box must be given its unique id"; - } - - if (options_.streaming_track_data_cache_size() > 0) { - RET_CHECK(!cc->InputSidePackets().HasTag("CACHE_DIR")) - << "Streaming mode not compatible with cache dir."; - } - - return absl::OkStatus(); -} - -absl::Status BoxTrackerCalculator::Process(CalculatorContext* cc) { - // Batch mode, issue tracking requests. - if (box_tracker_ && !tracking_issued_) { - for (const auto& pos : initial_pos_.box()) { - box_tracker_->NewBoxTrack(TimedBox::FromProto(pos), pos.id()); - } - tracking_issued_ = true; - } - - const Timestamp& timestamp = cc->InputTimestamp(); - if (timestamp == Timestamp::PreStream()) { - // Indicator packet. - return absl::OkStatus(); - } - - InputStream* track_stream = cc->Inputs().HasTag("TRACKING") - ? &(cc->Inputs().Tag("TRACKING")) - : nullptr; - InputStream* track_time_stream = cc->Inputs().HasTag("TRACK_TIME") - ? &(cc->Inputs().Tag("TRACK_TIME")) - : nullptr; - - // Cache tracking data if possible. - if (track_stream && !track_stream->IsEmpty()) { - const TrackingData& track_data = track_stream->Get(); - const int track_cache_size = options_.streaming_track_data_cache_size(); - if (track_cache_size > 0) { - tracking_data_cache_.push_back(std::make_pair(timestamp, track_data)); - while (tracking_data_cache_.size() > track_cache_size) { - tracking_data_cache_.pop_front(); - } - } - track_timestamps_.push_back(timestamp); - int trim_count = track_timestamps_.size() - - std::max(track_cache_size, kTrackTimestampsMinQueueSize); - if (trim_count > 0) { - track_timestamps_base_index_ += trim_count; - - while (trim_count-- > 0) { - track_timestamps_.pop_front(); - } - } - } - - InputStream* start_pos_stream = cc->Inputs().HasTag("START_POS") - ? &(cc->Inputs().Tag("START_POS")) - : nullptr; - - MotionBoxMap fast_forward_boxes; - if (start_pos_stream && !start_pos_stream->IsEmpty()) { - // Try to fast forward boxes to current tracking head. - const TimedBoxProtoList& start_pos_list = - start_pos_stream->Get(); - FastForwardStartPos(start_pos_list, &fast_forward_boxes); - } - - InputStream* start_pos_proto_string_stream = - cc->Inputs().HasTag("START_POS_PROTO_STRING") - ? &(cc->Inputs().Tag("START_POS_PROTO_STRING")) - : nullptr; - if (start_pos_stream == nullptr || start_pos_stream->IsEmpty()) { - if (start_pos_proto_string_stream && - !start_pos_proto_string_stream->IsEmpty()) { - auto start_pos_list_str = - start_pos_proto_string_stream->Get(); - TimedBoxProtoList start_pos_list; - start_pos_list.ParseFromString(start_pos_list_str); - FastForwardStartPos(start_pos_list, &fast_forward_boxes); - } - } - - InputStream* restart_pos_stream = cc->Inputs().HasTag("RESTART_POS") - ? &(cc->Inputs().Tag("RESTART_POS")) - : nullptr; - - if (restart_pos_stream && !restart_pos_stream->IsEmpty()) { - const TimedBoxProtoList& restart_pos_list = - restart_pos_stream->Get(); - FastForwardStartPos(restart_pos_list, &fast_forward_boxes); - } - - InputStream* cancel_object_id_stream = - cc->Inputs().HasTag("CANCEL_OBJECT_ID") - ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID")) - : nullptr; - if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) { - const int cancel_object_id = cancel_object_id_stream->Get(); - if (streaming_motion_boxes_.erase(cancel_object_id) == 0) { - LOG(WARNING) << "box id " << cancel_object_id << " does not exist."; - } - } - - cv::Mat input_view; - cv::Mat viz_view; - std::unique_ptr viz_frame; - - TrackingData track_data_to_render; - - if (cc->Outputs().HasTag("VIZ")) { - InputStream* video_stream = &(cc->Inputs().Tag("VIDEO")); - if (!video_stream->IsEmpty()) { - input_view = formats::MatView(&video_stream->Get()); - - viz_frame.reset(new ImageFrame()); - viz_frame->CopyFrom(video_stream->Get(), 16); - viz_view = formats::MatView(viz_frame.get()); - } - } - - // Results to be output or rendered, list of TimedBox for every id that are - // present at this frame. - TimedBoxProtoList box_track_list; - - CHECK(box_tracker_ || track_stream) - << "Expected either batch or streaming mode"; - - // Corresponding list of box states for rendering. For each id present at - // this frame stores closest 1-2 states. - std::vector> box_state_list; - int64 timestamp_msec = timestamp.Value() / 1000; - - if (box_tracker_) { // Batch mode. - // Ensure tracking has terminated. - box_tracker_->WaitForAllOngoingTracks(); - - // Cycle through ids. - for (int id : batch_track_ids_) { - TimedBox result; - std::vector states; - std::vector* states_ptr = - (visualize_state_ || visualize_internal_state_) ? &states : nullptr; - - if (box_tracker_->GetTimedPosition(id, timestamp_msec, &result, - states_ptr)) { - TimedBoxProto proto = result.ToProto(); - proto.set_id(id); - *box_track_list.add_box() = std::move(proto); - - if (states_ptr) { - box_state_list.push_back(*states_ptr); - } - } - } - - if (visualize_tracking_data_) { - constexpr int kVizId = -1; - box_tracker_->GetTrackingData(kVizId, timestamp_msec, - &track_data_to_render); - } - } else { - // Streaming mode. - // If track data is available advance all boxes by new data. - if (!track_stream->IsEmpty()) { - const TrackingData& track_data = track_stream->Get(); - - if (visualize_tracking_data_) { - track_data_to_render = track_data; - } - - const int64 time_ms = track_timestamps_.back().Value() / 1000; - const int64 duration_ms = - track_timestamps_.size() > 1 - ? time_ms - track_timestamps_.rbegin()[1].Value() / 1000 - : 0; - - std::vector failed_boxes; - StreamTrack(track_data, frame_num_, time_ms, duration_ms, - true, // forward. - &streaming_motion_boxes_, &failed_boxes); - - // Add fast forward boxes. - if (!fast_forward_boxes.empty()) { - for (const auto& box : fast_forward_boxes) { - streaming_motion_boxes_.emplace(box.first, box.second); - } - fast_forward_boxes.clear(); - } - - // Remove failed boxes. - for (int id : failed_boxes) { - streaming_motion_boxes_.erase(id); - } - - // Init new boxes once data from previous time to current is available. - for (const auto& pos : initial_pos_.box()) { - if (timestamp_msec - pos.time_msec() >= 0 && - initialized_ids_.find(pos.id()) == initialized_ids_.end()) { - MotionBoxState init_state; - MotionBoxStateFromTimedBox(TimedBox::FromProto(pos), &init_state); - - InitializeInliersOutliersInMotionBoxState(track_data, &init_state); - InitializePnpHomographyInMotionBoxState( - track_data, options_.tracker_options().track_step_options(), - &init_state); - - TrackStepOptions track_step_options = - options_.tracker_options().track_step_options(); - ChangeTrackingDegreesBasedOnStartPos(pos, &track_step_options); - MotionBox init_box(track_step_options); - - // Init at previous frame. - init_box.ResetAtFrame(frame_num_, init_state); - - PathSegment init_path; - AddStateToPath(init_state, timestamp_msec, &init_path); - - streaming_motion_boxes_.emplace( - pos.id(), MotionBoxPath(std::move(init_box), std::move(init_path), - pos.reacquisition())); - initialized_ids_.insert(pos.id()); - } - } - - ++frame_num_; - } else { - // Track stream is empty, if anything is requested on track_time_stream - // queue up requests. - if (track_time_stream && !track_time_stream->IsEmpty()) { - queued_track_requests_.push_back(timestamp); - } - } - - // Can output be generated? - if (!track_stream->IsEmpty()) { - ++frame_num_since_reset_; - - // Generate results for queued up request. - if (cc->Outputs().HasTag("BOXES") && !queued_track_requests_.empty()) { - for (int j = 0; j < queued_track_requests_.size(); ++j) { - const Timestamp& past_time = queued_track_requests_[j]; - RET_CHECK(past_time.Value() < timestamp.Value()) - << "Inconsistency, queued up requests should occur in past"; - std::unique_ptr past_box_list( - new TimedBoxProtoList()); - - for (auto& motion_box_path : streaming_motion_boxes_) { - TimedBox result_box; - TimedBoxAtTime(motion_box_path.second.path, - past_time.Value() / 1000, &result_box, nullptr); - - const float subframe_alpha = - static_cast(j + 1) / (queued_track_requests_.size() + 1); - AddSmoothTransitionToOutputBox(motion_box_path.first, &result_box, - subframe_alpha); - - TimedBoxProto proto = result_box.ToProto(); - proto.set_id(motion_box_path.first); - proto.set_reacquisition(motion_box_path.second.reacquisition); - *past_box_list->add_box() = std::move(proto); - } - - // Output for every time. - cc->Outputs().Tag("BOXES").Add(past_box_list.release(), past_time); - } - - queued_track_requests_.clear(); - } - - // Generate result at current frame. - for (auto& motion_box_path : streaming_motion_boxes_) { - TimedBox result_box; - MotionBoxState result_state; - TimedBoxAtTime(motion_box_path.second.path, timestamp_msec, &result_box, - &result_state); - - AddSmoothTransitionToOutputBox(motion_box_path.first, &result_box); - - TimedBoxProto proto = result_box.ToProto(); - proto.set_id(motion_box_path.first); - proto.set_reacquisition(motion_box_path.second.reacquisition); - *box_track_list.add_box() = std::move(proto); - - if (visualize_state_ || visualize_internal_state_) { - box_state_list.push_back({result_state}); - } - } - } - // end streaming mode case. - } - - // Save a snapshot of latest tracking results before override with fast - // forwarded start pos. - if (!fast_forward_boxes.empty()) { - frame_num_since_reset_ = 0; - last_tracked_boxes_.clear(); - // Add any remaining fast forward boxes. For example occurs if START_POS is - // specified with non-matching TRACKING mode - for (const auto& reset_box : fast_forward_boxes) { - const auto tracked_box_iter = - streaming_motion_boxes_.find(reset_box.first); - if (tracked_box_iter != streaming_motion_boxes_.end()) { - if (!reset_box.second.path.empty() && - !tracked_box_iter->second.path.empty()) { - last_tracked_boxes_[reset_box.first] = - std::make_pair(tracked_box_iter->second.path.back(), - reset_box.second.path.back()); - } - } - - // Override previous tracking with reset start pos. - streaming_motion_boxes_[reset_box.first] = reset_box.second; - } - } - - if (viz_frame) { - if (visualize_tracking_data_) { - RenderTrackingData(track_data_to_render, &viz_view); - } - - if (visualize_state_) { - for (const auto& state_vec : box_state_list) { - RenderStates(state_vec, &viz_view); - } - } - - if (visualize_internal_state_) { - for (const auto& state_vec : box_state_list) { - RenderInternalStates(state_vec, &viz_view); - } - } - - for (const auto& box : box_track_list.box()) { - RenderBox(box, &viz_view); - } - } - - // Handle random access track requests. - InputStream* ra_track_stream = cc->Inputs().HasTag("RA_TRACK") - ? &(cc->Inputs().Tag("RA_TRACK")) - : nullptr; - - if (ra_track_stream && !ra_track_stream->IsEmpty()) { - RET_CHECK(!box_tracker_) << "Random access only for streaming mode " - << "implemented."; - const TimedBoxProtoList& box_list = - ra_track_stream->Get(); - RET_CHECK(box_list.box_size() % 2 == 0) - << "Expect even number of (start,end) tuples but get " - << box_list.box_size(); - OutputRandomAccessTrack(box_list, cc); - } - - InputStream* ra_track_proto_string_stream = - cc->Inputs().HasTag("RA_TRACK_PROTO_STRING") - ? &(cc->Inputs().Tag("RA_TRACK_PROTO_STRING")) - : nullptr; - if (ra_track_stream == nullptr || ra_track_stream->IsEmpty()) { - if (ra_track_proto_string_stream && - !ra_track_proto_string_stream->IsEmpty()) { - RET_CHECK(!box_tracker_) << "Random access only for streaming mode " - << "implemented."; - auto box_list_str = ra_track_proto_string_stream->Get(); - TimedBoxProtoList box_list; - box_list.ParseFromString(box_list_str); - RET_CHECK(box_list.box_size() % 2 == 0) - << "Expect even number of (start,end) tuples but get " - << box_list.box_size(); - OutputRandomAccessTrack(box_list, cc); - } - } - - // Always output in batch, only output in streaming if tracking data - // is present (might be in fast forward mode instead). - if (cc->Outputs().HasTag("BOXES") && - (box_tracker_ || !track_stream->IsEmpty())) { - std::unique_ptr boxes(new TimedBoxProtoList()); - *boxes = std::move(box_track_list); - cc->Outputs().Tag("BOXES").Add(boxes.release(), timestamp); - } - - if (viz_frame) { - cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp); - } - - return absl::OkStatus(); -} - -void BoxTrackerCalculator::AddSmoothTransitionToOutputBox( - int box_id, TimedBox* result_box, float subframe_alpha) { - if (options_.start_pos_transition_frames() > 0 && - frame_num_since_reset_ <= options_.start_pos_transition_frames()) { - const auto& box_iter = last_tracked_boxes_.find(box_id); - if (box_iter != last_tracked_boxes_.end()) { - // We first compute the blend of last tracked box with reset box at the - // same timestamp as blend_start = alpha * reset_box + (1 - alpha) * - // last_tracked_box. Then apply the motion from current tracking to reset - // pos to the blended start pos as: result_box = blend_start + - // (current_box - reset_box) With some derivation, we can get result_box = - // (1 - alpha) * (last_track - reset_box) + current_box - TimedBox tmp_box = TimedBox::Blend(box_iter->second.first, - box_iter->second.second, 1.0, -1.0); - const float alpha = (frame_num_since_reset_ - 1.0f + subframe_alpha) / - options_.start_pos_transition_frames(); - *result_box = TimedBox::Blend(tmp_box, *result_box, 1.0 - alpha, 1.0); - } - } -} - -void BoxTrackerCalculator::OutputRandomAccessTrack( - const TimedBoxProtoList& box_list, CalculatorContext* cc) { - std::unique_ptr result_list(new TimedBoxProtoList()); - - for (int i = 0; i < box_list.box_size(); i += 2) { - const TimedBoxProto start = box_list.box(i); - int64 end_time_msec = box_list.box(i + 1).time_msec(); - const bool forward_track = start.time_msec() < end_time_msec; - - if (track_timestamps_.empty()) { - LOG(WARNING) << "No tracking data cached yet."; - continue; - } - - // Performing the range check in msec (b/138399787) - const int64 tracking_start_timestamp_msec = - track_timestamps_.front().Microseconds() / 1000; - const int64 tracking_end_timestamp_msec = - track_timestamps_.back().Microseconds() / 1000; - if (start.time_msec() < tracking_start_timestamp_msec) { - LOG(WARNING) << "Request start timestamp " << start.time_msec() - << " too old. First frame in the window: " - << tracking_start_timestamp_msec; - continue; - } - if (start.time_msec() > tracking_end_timestamp_msec) { - LOG(WARNING) << "Request start timestamp " << start.time_msec() - << " too new. Last frame in the window: " - << tracking_end_timestamp_msec; - continue; - } - if (end_time_msec < tracking_start_timestamp_msec) { - LOG(WARNING) << "Request end timestamp " << end_time_msec - << " too old. First frame in the window: " - << tracking_start_timestamp_msec; - continue; - } - if (end_time_msec > tracking_end_timestamp_msec) { - LOG(WARNING) << "Request end timestamp " << end_time_msec - << " too new. Last frame in the window: " - << tracking_end_timestamp_msec; - continue; - } - - std::deque::iterator timestamp_pos = - GetRandomAccessTimestampPos(start, forward_track); - - if (timestamp_pos == track_timestamps_.end()) { - LOG(ERROR) << "Random access outside cached range"; - continue; - } - - // Locate start of tracking data. - std::deque>::iterator start_data = - GetRandomAccessStartData(timestamp_pos); - - // TODO: Interpolate random access tracking start_data instead - // of dropping the request in the case of missing processed frame. - if (start_data == tracking_data_cache_.end()) { - LOG(ERROR) << "Random access starts at unprocessed frame."; - continue; - } - - const int init_frame = timestamp_pos - track_timestamps_.begin() + - track_timestamps_base_index_; - CHECK_GE(init_frame, 0); - - MotionBoxMap single_map = - PrepareRandomAccessTrack(start, init_frame, forward_track, start_data); - bool track_error = forward_track - ? RunForwardTrack(start_data, init_frame, - &single_map, end_time_msec) - : RunBackwardTrack(start_data, init_frame, - &single_map, end_time_msec); - - if (track_error) { - LOG(ERROR) << "Could not track box."; - continue; - } - - ObtainResultOfRandomAccessTrack(single_map, start, end_time_msec, - result_list); - } - - cc->Outputs() - .Tag("RA_BOXES") - .Add(result_list.release(), cc->InputTimestamp()); -} - -std::deque::iterator -BoxTrackerCalculator::GetRandomAccessTimestampPos(const TimedBoxProto& start, - bool forward_track) { - std::deque::iterator timestamp_pos; - Timestamp timestamp(start.time_msec() * 1000); - if (forward_track) { - timestamp_pos = std::upper_bound(track_timestamps_.begin(), - track_timestamps_.end(), timestamp); - } else { - timestamp_pos = std::lower_bound(track_timestamps_.begin(), - track_timestamps_.end(), timestamp); - } - return timestamp_pos; -} - -std::deque>::iterator -BoxTrackerCalculator::GetRandomAccessStartData( - const std::deque::iterator& timestamp_pos) { - std::deque>::iterator start_data = - std::find_if(tracking_data_cache_.begin(), tracking_data_cache_.end(), - [timestamp_pos]( - const std::pair& item) -> bool { - return item.first == *timestamp_pos; - }); - return start_data; -} - -BoxTrackerCalculator::MotionBoxMap -BoxTrackerCalculator::PrepareRandomAccessTrack( - const TimedBoxProto& start, int init_frame, bool forward_track, - const std::deque>::iterator& - start_data) { - MotionBoxMap single_map; - // Init state at request time. - MotionBoxState init_state; - MotionBoxStateFromTimedBox(TimedBox::FromProto(start), &init_state); - - InitializeInliersOutliersInMotionBoxState(start_data->second, &init_state); - InitializePnpHomographyInMotionBoxState( - start_data->second, options_.tracker_options().track_step_options(), - &init_state); - - TrackStepOptions track_step_options = - options_.tracker_options().track_step_options(); - ChangeTrackingDegreesBasedOnStartPos(start, &track_step_options); - MotionBox init_box(track_step_options); - init_box.ResetAtFrame(init_frame - (forward_track ? 1 : 0), init_state); - - PathSegment init_path; - - // Avoid duplicating start time in case TrackingData has same value. - // Note: For backward tracking we always arrive at an earlier frame, so - // no duplication can happen, see StreamTrack for details. - if (start.time_msec() != start_data->first.Value() / 1000 || !forward_track) { - AddStateToPath(init_state, start.time_msec(), &init_path); - } - - single_map.emplace(start.id(), - MotionBoxPath(std::move(init_box), std::move(init_path))); - return single_map; -} - -bool BoxTrackerCalculator::RunForwardTrack( - const std::deque>::iterator& start_data, - int init_frame, MotionBoxMap* single_map, int64 end_time_msec) { - int curr_frame = init_frame; - for (auto cache_pos = start_data; cache_pos != tracking_data_cache_.end(); - ++cache_pos, ++curr_frame) { - std::vector failed_box; - const int64 dst_time_msec = cache_pos->first.Value() / 1000; - const int64 curr_duration = - (cache_pos == tracking_data_cache_.begin()) - ? 0 - : (cache_pos[0].first.Value() - cache_pos[-1].first.Value()) / 1000; - StreamTrack(cache_pos->second, curr_frame, dst_time_msec, curr_duration, - true, // forward - single_map, &failed_box); - if (!failed_box.empty()) { - return true; - } - if (dst_time_msec > end_time_msec) { - return false; - } - } - return false; -} - -bool BoxTrackerCalculator::RunBackwardTrack( - const std::deque>::iterator& start_data, - int init_frame, MotionBoxMap* single_map, int64 end_time_msec) { - int curr_frame = init_frame; - for (auto cache_pos = start_data; cache_pos != tracking_data_cache_.begin(); - --cache_pos, --curr_frame) { - std::vector failed_box; - const int64 dst_time_msec = cache_pos[-1].first.Value() / 1000; - const int64 curr_duration = - (cache_pos[0].first.Value() - cache_pos[-1].first.Value()) / 1000; - StreamTrack(cache_pos->second, curr_frame, dst_time_msec, curr_duration, - false, // backward - single_map, &failed_box); - if (!failed_box.empty()) { - return true; - } - if (dst_time_msec < end_time_msec) { - return false; - } - } - return false; -} - -void BoxTrackerCalculator::ObtainResultOfRandomAccessTrack( - const MotionBoxMap& single_map, const TimedBoxProto& start, - int64 end_time_msec, - const std::unique_ptr& result_list) { - const MotionBoxPath& result_path = single_map.find(start.id())->second; - TimedBox result_box; - TimedBoxAtTime(result_path.path, end_time_msec, &result_box, nullptr); - TimedBoxProto proto = result_box.ToProto(); - proto.set_id(start.id()); - *result_list->add_box() = std::move(proto); -} - -void BoxTrackerCalculator::RenderStates( - const std::vector& states, cv::Mat* mat) { - for (int k = 0; k < states.size(); ++k) { - const bool print_stats = k == 0; - RenderState(states[k], print_stats, mat); - } -} - -void BoxTrackerCalculator::RenderInternalStates( - const std::vector& states, cv::Mat* mat) { - for (const MotionBoxState& state : states) { - RenderInternalState(state.internal(), mat); - } -} - -void BoxTrackerCalculator::StreamTrack(const TrackingData& data, - int data_frame_num, - int64 dst_timestamp_ms, - int64 duration_ms, bool forward, - MotionBoxMap* box_map, - std::vector* failed_ids) { - CHECK(box_map); - CHECK(failed_ids); - - // Cache the actively discarded tracked ids from the new tracking data. - for (const int discarded_id : - data.motion_data().actively_discarded_tracked_ids()) { - actively_discarded_tracked_ids_.insert(discarded_id); - } - - // Track all existing boxes by one frame. - MotionVectorFrame mvf; // Holds motion from current to previous frame. - MotionVectorFrameFromTrackingData(data, &mvf); - mvf.actively_discarded_tracked_ids = &actively_discarded_tracked_ids_; - - if (forward) { - MotionVectorFrame mvf_inverted; - InvertMotionVectorFrame(mvf, &mvf_inverted); - std::swap(mvf, mvf_inverted); - } - - if (duration_ms > 0) { - mvf.duration_ms = duration_ms; - } - - const int from_frame = data_frame_num - (forward ? 1 : 0); - const int to_frame = forward ? from_frame + 1 : from_frame - 1; - - for (auto& motion_box : *box_map) { - if (!motion_box.second.box.TrackStep(from_frame, // from frame. - mvf, forward)) { - failed_ids->push_back(motion_box.first); - LOG(INFO) << "lost track. pushed failed id: " << motion_box.first; - } else { - // Store result. - PathSegment& path = motion_box.second.path; - const MotionBoxState& result_state = - motion_box.second.box.StateAtFrame(to_frame); - AddStateToPath(result_state, dst_timestamp_ms, &path); - // motion_box has got new tracking state/path. Now trimming it. - const int cache_size = - std::max(options_.streaming_track_data_cache_size(), - kMotionBoxPathMinQueueSize); - motion_box.second.Trim(cache_size, forward); - } - } -} - -void BoxTrackerCalculator::FastForwardStartPos( - const TimedBoxProtoList& start_pos_list, MotionBoxMap* box_map) { - for (const TimedBoxProto& start_pos : start_pos_list.box()) { - Timestamp timestamp(start_pos.time_msec() * 1000); - // Locate corresponding frame number for starting position. As TrackingData - // stores motion from current to last frame; we are using the data after - // this frame for tracking. - auto timestamp_pos = std::lower_bound(track_timestamps_.begin(), - track_timestamps_.end(), timestamp); - - if (timestamp_pos == track_timestamps_.end()) { - LOG(WARNING) << "Received start pos beyond current timestamp, " - << "Starting to track once frame arrives."; - *initial_pos_.add_box() = start_pos; - continue; - } - - // Start at previous frame. - const int init_frame = timestamp_pos - track_timestamps_.begin() + - track_timestamps_base_index_; - CHECK_GE(init_frame, 0); - - // Locate corresponding tracking data. - auto start_data = std::find_if( - tracking_data_cache_.begin(), tracking_data_cache_.end(), - [timestamp_pos](const std::pair& item) - -> bool { return item.first == timestamp_pos[0]; }); - - if (start_data == tracking_data_cache_.end()) { - LOG(ERROR) << "Box to fast forward outside tracking data cache. Ignoring." - << " To avoid this error consider increasing the cache size."; - continue; - } - - // Init state at request time. - MotionBoxState init_state; - MotionBoxStateFromTimedBox(TimedBox::FromProto(start_pos), &init_state); - - InitializeInliersOutliersInMotionBoxState(start_data->second, &init_state); - InitializePnpHomographyInMotionBoxState( - start_data->second, options_.tracker_options().track_step_options(), - &init_state); - - TrackStepOptions track_step_options = - options_.tracker_options().track_step_options(); - ChangeTrackingDegreesBasedOnStartPos(start_pos, &track_step_options); - MotionBox init_box(track_step_options); - init_box.ResetAtFrame(init_frame, init_state); - - int curr_frame = init_frame + 1; - MotionBoxMap single_map; - PathSegment init_path; - AddStateToPath(init_state, timestamp_pos[0].Value() / 1000, &init_path); - single_map.emplace(start_pos.id(), - MotionBoxPath(std::move(init_box), std::move(init_path), - start_pos.reacquisition())); - bool track_error = false; - - for (auto cache_pos = start_data + 1; - cache_pos != tracking_data_cache_.end(); ++cache_pos, ++curr_frame) { - std::vector failed_box; - const int64 curr_time_msec = cache_pos->first.Value() / 1000; - const int64 curr_duration = - (cache_pos[0].first.Value() - cache_pos[-1].first.Value()) / 1000; - StreamTrack(cache_pos->second, curr_frame, curr_time_msec, curr_duration, - true, // forward - &single_map, &failed_box); - if (!failed_box.empty()) { - LOG(WARNING) << "Unable to fast forward box at frame " << curr_frame; - track_error = true; - break; - } - } - - if (!track_error) { - // Fast forward successful. - if (box_map->find(start_pos.id()) != box_map->end()) { - DLOG(ERROR) << "Fast forward successful, but box with same id " - << "exists already."; - } else { - // Add to set of currently tracked boxes. - const MotionBoxPath& result = single_map.find(start_pos.id())->second; - box_map->emplace(start_pos.id(), result); - } - } - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/box_tracker_calculator.proto b/mediapipe/calculators/video/box_tracker_calculator.proto deleted file mode 100644 index 200108201..000000000 --- a/mediapipe/calculators/video/box_tracker_calculator.proto +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/tracking/box_tracker.proto"; - -message BoxTrackerCalculatorOptions { - extend CalculatorOptions { - optional BoxTrackerCalculatorOptions ext = 268767860; - } - - optional BoxTrackerOptions tracker_options = 1; - - // Initial position to be tracked. Can also be supplied as side packet or - // as input stream. - optional TimedBoxProtoList initial_position = 2; - - // If set and VIZ stream is present, renders tracking data into the - // visualization. - optional bool visualize_tracking_data = 3 [default = false]; - - // If set and VIZ stream is present, renders the box state - // into the visualization. - optional bool visualize_state = 4 [default = false]; - - // If set and VIZ stream is present, renders the internal box state - // into the visualization. - optional bool visualize_internal_state = 5 [default = false]; - - // Size of the track data cache during streaming mode. This allows to buffer - // track_data's for fast forward tracking, i.e. any TimedBox received - // via input stream START_POS can be tracked towards the current track head - // (i.e. last received TrackingData). Measured in number of frames. - optional int32 streaming_track_data_cache_size = 6 [default = 0]; - - // Add a transition period of N frames to smooth the jump from original - // tracking to reset start pos with motion compensation. The transition will - // be a linear decay of original tracking result. 0 means no transition. - optional int32 start_pos_transition_frames = 7 [default = 0]; -} diff --git a/mediapipe/calculators/video/flow_packager_calculator.cc b/mediapipe/calculators/video/flow_packager_calculator.cc deleted file mode 100644 index a57105928..000000000 --- a/mediapipe/calculators/video/flow_packager_calculator.cc +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/video/flow_packager_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/util/tracking/camera_motion.pb.h" -#include "mediapipe/util/tracking/flow_packager.h" -#include "mediapipe/util/tracking/region_flow.pb.h" - -namespace mediapipe { - -using mediapipe::CameraMotion; -using mediapipe::FlowPackager; -using mediapipe::RegionFlowFeatureList; -using mediapipe::TrackingData; -using mediapipe::TrackingDataChunk; - -// A calculator that packages input CameraMotion and RegionFlowFeatureList -// into a TrackingData and optionally writes TrackingDataChunks to file. -// -// Input stream: -// FLOW: Input region flow (proto RegionFlowFeatureList). -// CAMERA: Input camera stream (proto CameraMotion, optional). -// -// Input side packets: -// CACHE_DIR: Optional caching directory tracking files are written to. -// -// Output streams. -// TRACKING: Output tracking data (proto TrackingData, per frame -// optional). -// TRACKING_CHUNK: Output tracking chunks (proto TrackingDataChunk, -// per chunk, optional), output at the first timestamp -// of each chunk. -// COMPLETE: Optional output packet sent on PreStream to -// to signal downstream calculators that all data has been -// processed and calculator is closed. Can be used to indicate -// that all data as been written to CACHE_DIR. -class FlowPackagerCalculator : public CalculatorBase { - public: - ~FlowPackagerCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - // Writes passed chunk to disk. - void WriteChunk(const TrackingDataChunk& chunk) const; - - // Initializes next chunk for tracking beginning from last frame of - // current chunk (Chunking is design with one frame overlap). - void PrepareCurrentForNextChunk(TrackingDataChunk* chunk); - - private: - FlowPackagerCalculatorOptions options_; - - // Caching options. - bool use_caching_ = false; - bool build_chunk_ = false; - std::string cache_dir_; - int chunk_idx_ = -1; - TrackingDataChunk tracking_chunk_; - - int frame_idx_ = 0; - - Timestamp prev_timestamp_; - std::unique_ptr flow_packager_; -}; - -REGISTER_CALCULATOR(FlowPackagerCalculator); - -absl::Status FlowPackagerCalculator::GetContract(CalculatorContract* cc) { - if (!cc->Inputs().HasTag("FLOW")) { - return tool::StatusFail("No input flow was specified."); - } - - cc->Inputs().Tag("FLOW").Set(); - - if (cc->Inputs().HasTag("CAMERA")) { - cc->Inputs().Tag("CAMERA").Set(); - } - if (cc->Outputs().HasTag("TRACKING")) { - cc->Outputs().Tag("TRACKING").Set(); - } - if (cc->Outputs().HasTag("TRACKING_CHUNK")) { - cc->Outputs().Tag("TRACKING_CHUNK").Set(); - } - if (cc->Outputs().HasTag("COMPLETE")) { - cc->Outputs().Tag("COMPLETE").Set(); - } - - if (cc->InputSidePackets().HasTag("CACHE_DIR")) { - cc->InputSidePackets().Tag("CACHE_DIR").Set(); - } - - return absl::OkStatus(); -} - -absl::Status FlowPackagerCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - - flow_packager_.reset(new FlowPackager(options_.flow_packager_options())); - - use_caching_ = cc->InputSidePackets().HasTag("CACHE_DIR"); - build_chunk_ = use_caching_ || cc->Outputs().HasTag("TRACKING_CHUNK"); - if (use_caching_) { - cache_dir_ = cc->InputSidePackets().Tag("CACHE_DIR").Get(); - } - - return absl::OkStatus(); -} - -absl::Status FlowPackagerCalculator::Process(CalculatorContext* cc) { - InputStream* flow_stream = &(cc->Inputs().Tag("FLOW")); - const RegionFlowFeatureList& flow = flow_stream->Get(); - - const Timestamp timestamp = flow_stream->Value().Timestamp(); - - const CameraMotion* camera_motion = nullptr; - if (cc->Inputs().HasTag("CAMERA")) { - InputStream* camera_stream = &(cc->Inputs().Tag("CAMERA")); - camera_motion = &camera_stream->Get(); - } - - std::unique_ptr tracking_data(new TrackingData()); - - flow_packager_->PackFlow(flow, camera_motion, tracking_data.get()); - - if (build_chunk_) { - if (chunk_idx_ < 0) { // Lazy init, determine first start. - chunk_idx_ = - timestamp.Value() / 1000 / options_.caching_chunk_size_msec(); - tracking_chunk_.set_first_chunk(true); - } - CHECK_GE(chunk_idx_, 0); - - TrackingDataChunk::Item* item = tracking_chunk_.add_item(); - item->set_frame_idx(frame_idx_); - item->set_timestamp_usec(timestamp.Value()); - if (frame_idx_ > 0) { - item->set_prev_timestamp_usec(prev_timestamp_.Value()); - } - if (cc->Outputs().HasTag("TRACKING")) { - // Need to copy as output is requested. - *item->mutable_tracking_data() = *tracking_data; - } else { - item->mutable_tracking_data()->Swap(tracking_data.get()); - } - - const int next_chunk_msec = - options_.caching_chunk_size_msec() * (chunk_idx_ + 1); - - if (timestamp.Value() / 1000 >= next_chunk_msec) { - if (cc->Outputs().HasTag("TRACKING_CHUNK")) { - cc->Outputs() - .Tag("TRACKING_CHUNK") - .Add(new TrackingDataChunk(tracking_chunk_), - Timestamp(tracking_chunk_.item(0).timestamp_usec())); - } - if (use_caching_) { - WriteChunk(tracking_chunk_); - } - PrepareCurrentForNextChunk(&tracking_chunk_); - } - } - - if (cc->Outputs().HasTag("TRACKING")) { - cc->Outputs() - .Tag("TRACKING") - .Add(tracking_data.release(), flow_stream->Value().Timestamp()); - } - - prev_timestamp_ = timestamp; - ++frame_idx_; - return absl::OkStatus(); -} - -absl::Status FlowPackagerCalculator::Close(CalculatorContext* cc) { - if (frame_idx_ > 0) { - tracking_chunk_.set_last_chunk(true); - if (cc->Outputs().HasTag("TRACKING_CHUNK")) { - cc->Outputs() - .Tag("TRACKING_CHUNK") - .Add(new TrackingDataChunk(tracking_chunk_), - Timestamp(tracking_chunk_.item(0).timestamp_usec())); - } - - if (use_caching_) { - WriteChunk(tracking_chunk_); - } - } - - if (cc->Outputs().HasTag("COMPLETE")) { - cc->Outputs().Tag("COMPLETE").Add(new bool(true), Timestamp::PreStream()); - } - - return absl::OkStatus(); -} - -void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const { - if (chunk.item_size() == 0) { - LOG(ERROR) << "Write chunk called with empty tracking data." - << "This can only occur if the spacing between frames " - << "is larger than the requested chunk size. Try increasing " - << "the chunk size"; - return; - } - - auto format_runtime = - absl::ParsedFormat<'d'>::New(options_.cache_file_format()); - - std::string chunk_file; - if (format_runtime) { - chunk_file = - cache_dir_ + "/" + absl::StrFormat(*format_runtime, chunk_idx_); - } else { - LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d."; - chunk_file = cache_dir_ + "/" + absl::StrFormat("chunk_%04d", chunk_idx_); - } - - std::string data; - chunk.SerializeToString(&data); - - const char* temp_filename = tempnam(cache_dir_.c_str(), nullptr); - std::ofstream out_file(temp_filename); - if (!out_file) { - LOG(ERROR) << "Could not open " << temp_filename; - } else { - out_file.write(data.data(), data.size()); - } - - if (rename(temp_filename, chunk_file.c_str()) != 0) { - LOG(ERROR) << "Failed to rename to " << chunk_file; - } - - LOG(INFO) << "Wrote chunk : " << chunk_file; -} - -void FlowPackagerCalculator::PrepareCurrentForNextChunk( - TrackingDataChunk* chunk) { - CHECK(chunk); - if (chunk->item_size() == 0) { - LOG(ERROR) << "Called with empty chunk. Unexpected."; - return; - } - - chunk->set_first_chunk(false); - - // Buffer last item for next chunk. - TrackingDataChunk::Item last_item; - last_item.Swap(chunk->mutable_item(chunk->item_size() - 1)); - - chunk->Clear(); - chunk->add_item()->Swap(&last_item); - - ++chunk_idx_; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/flow_packager_calculator.proto b/mediapipe/calculators/video/flow_packager_calculator.proto deleted file mode 100644 index c1e91d8c5..000000000 --- a/mediapipe/calculators/video/flow_packager_calculator.proto +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/tracking/flow_packager.proto"; - -message FlowPackagerCalculatorOptions { - extend CalculatorOptions { - optional FlowPackagerCalculatorOptions ext = 271236147; - } - - optional mediapipe.FlowPackagerOptions flow_packager_options = 1; - - // Chunk size for caching files that are written to the externally specified - // caching directory. Specified in msec. - // Note that each chunk always contains at its end the first frame of the - // next chunk (to enable forward tracking across chunk boundaries). - optional int32 caching_chunk_size_msec = 2 [default = 2500]; - - optional string cache_file_format = 3 [default = "chunk_%04d"]; -} diff --git a/mediapipe/calculators/video/flow_to_image_calculator.cc b/mediapipe/calculators/video/flow_to_image_calculator.cc deleted file mode 100644 index 6a078ee72..000000000 --- a/mediapipe/calculators/video/flow_to_image_calculator.cc +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// MediaPipe calculator to take a flow field as input, and outputs a normalized -// RGB image where the B channel is forced to zero. -// TODO: Add video stream header for visualization - -#include -#include -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/video/flow_to_image_calculator.pb.h" -#include "mediapipe/calculators/video/tool/flow_quantizer_model.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/motion/optical_flow_field.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" - -namespace mediapipe { - -// Reads optical flow fields defined in -// mediapipe/framework/formats/motion/optical_flow_field.h, -// returns a VideoFrame with 2 channels (v_x and v_y), each channel is quantized -// to 0-255. -// -// Example config: -// node { -// calculator: "FlowToImageCalculator" -// input_stream: "flow_fields" -// output_stream: "frames" -// options: { -// [type.googleapis.com/mediapipe.FlowToImageCalculatorOptions]:{ -// min_value: -40.0 -// max_value: 40.0 -// } -// } -// } -class FlowToImageCalculator : public CalculatorBase { - public: - FlowToImageCalculator() {} - ~FlowToImageCalculator() override {} - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - FlowQuantizerModel model_; -}; - -absl::Status FlowToImageCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - cc->Outputs().Index(0).Set(); - - // Model sanity check - const auto& options = cc->Options(); - if (options.min_value() >= options.max_value()) { - return absl::InvalidArgumentError("Invalid quantizer model."); - } - return absl::OkStatus(); -} - -absl::Status FlowToImageCalculator::Open(CalculatorContext* cc) { - const auto& options = cc->Options(); - // Fill the the model_data, ideally we want to train the model, but we omit - // the step for now, and takes the (min, max) range from protobuf. - const QuantizerModelData& model_data = - ParseTextProtoOrDie( - absl::StrFormat("min_value:%f min_value:%f max_value:%f max_value:%f", - options.min_value(), options.min_value(), - options.max_value(), options.max_value())); - model_.LoadFromProto(model_data); - return absl::OkStatus(); -} - -absl::Status FlowToImageCalculator::Process(CalculatorContext* cc) { - const auto& input = cc->Inputs().Index(0).Get(); - // Input flow is 2-channel with x-dim flow and y-dim flow. - // Convert it to a ImageFrame in SRGB space, the 3rd channel is not used (0). - const cv::Mat_& flow = input.flow_data(); - std::unique_ptr output( - new ImageFrame(ImageFormat::SRGB, input.width(), input.height())); - cv::Mat image = ::mediapipe::formats::MatView(output.get()); - - for (int j = 0; j != input.height(); ++j) { - for (int i = 0; i != input.width(); ++i) { - image.at(j, i) = - cv::Vec3b(model_.Apply(flow.at(j, i).x, 0), - model_.Apply(flow.at(j, i).y, 1), 0); - } - } - cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(FlowToImageCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/flow_to_image_calculator.proto b/mediapipe/calculators/video/flow_to_image_calculator.proto deleted file mode 100644 index 6d5fb8450..000000000 --- a/mediapipe/calculators/video/flow_to_image_calculator.proto +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -// Specifies the maximum and minimum value to truncate when normalize optical -// flow fields. -message FlowToImageCalculatorOptions { - extend CalculatorOptions { - optional FlowToImageCalculatorOptions ext = 69508592; - } - optional float min_value = 1 [default = -40.0]; - optional float max_value = 2 [default = 40.0]; -} diff --git a/mediapipe/calculators/video/motion_analysis_calculator.cc b/mediapipe/calculators/video/motion_analysis_calculator.cc deleted file mode 100644 index 59673108c..000000000 --- a/mediapipe/calculators/video/motion_analysis_calculator.cc +++ /dev/null @@ -1,987 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/strings/numbers.h" -#include "absl/strings/str_split.h" -#include "absl/strings/string_view.h" -#include "mediapipe/calculators/video/motion_analysis_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/tracking/camera_motion.h" -#include "mediapipe/util/tracking/camera_motion.pb.h" -#include "mediapipe/util/tracking/frame_selection.pb.h" -#include "mediapipe/util/tracking/motion_analysis.h" -#include "mediapipe/util/tracking/motion_estimation.h" -#include "mediapipe/util/tracking/motion_models.h" -#include "mediapipe/util/tracking/region_flow.pb.h" - -namespace mediapipe { - -using mediapipe::AffineAdapter; -using mediapipe::CameraMotion; -using mediapipe::FrameSelectionResult; -using mediapipe::Homography; -using mediapipe::HomographyAdapter; -using mediapipe::LinearSimilarityModel; -using mediapipe::MixtureHomography; -using mediapipe::MixtureRowWeights; -using mediapipe::MotionAnalysis; -using mediapipe::ProjectViaFit; -using mediapipe::RegionFlowComputationOptions; -using mediapipe::RegionFlowFeatureList; -using mediapipe::SalientPointFrame; -using mediapipe::TranslationModel; - -const char kOptionsTag[] = "OPTIONS"; - -// A calculator that performs motion analysis on an incoming video stream. -// -// Input streams: (at least one of them is required). -// VIDEO: The input video stream (ImageFrame, sRGB, sRGBA or GRAY8). -// SELECTION: Optional input stream to perform analysis only on selected -// frames. If present needs to contain camera motion -// and features. -// -// Input side packets: -// CSV_FILE: Read motion models as homographies from CSV file. Expected -// to be defined in the frame domain (un-normalized). -// Should store 9 floats per row. -// Specify number of homographies per frames via option -// meta_models_per_frame. For values > 1, MixtureHomographies -// are created, for value == 1, a single Homography is used. -// DOWNSAMPLE: Optionally specify downsampling factor via input side packet -// overriding value in the graph settings. -// Output streams (all are optional). -// FLOW: Sparse feature tracks in form of proto RegionFlowFeatureList. -// CAMERA: Camera motion as proto CameraMotion describing the per frame- -// pair motion. Has VideoHeader from input video. -// SALIENCY: Foreground saliency (objects moving different from the -// background) as proto SalientPointFrame. -// VIZ: Visualization stream as ImageFrame, sRGB, visualizing -// features and saliency (set via -// analysis_options().visualization_options()) -// DENSE_FG: Dense foreground stream, describing per-pixel foreground- -// ness as confidence between 0 (background) and 255 -// (foreground). Output is ImageFrame (GRAY8). -// VIDEO_OUT: Optional output stream when SELECTION is used. Output is input -// VIDEO at the selected frames. Required VIDEO to be present. -// GRAY_VIDEO_OUT: Optional output stream for downsampled, grayscale video. -// Requires VIDEO to be present and SELECTION to not be used. -class MotionAnalysisCalculator : public CalculatorBase { - // TODO: Activate once leakr approval is ready. - // typedef com::google::android::libraries::micro::proto::Data HomographyData; - - public: - ~MotionAnalysisCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - // Outputs results to Outputs() if MotionAnalysis buffered sufficient results. - // Otherwise no-op. Set flush to true to force output of all buffered data. - void OutputMotionAnalyzedFrames(bool flush, CalculatorContext* cc); - - // Lazy init function to be called on Process. - absl::Status InitOnProcess(InputStream* video_stream, - InputStream* selection_stream); - - // Parses CSV file contents to homographies. - bool ParseModelCSV(const std::string& contents, - std::deque* homographies); - - // Turns list of 9-tuple floating values into set of homographies. - bool HomographiesFromValues(const std::vector& homog_values, - std::deque* homographies); - - // Appends CameraMotions and features from homographies. - // Set append_identity to true to add an identity transform to the beginning - // of the each list *in addition* to the motions derived from homographies. - void AppendCameraMotionsFromHomographies( - const std::deque& homographies, bool append_identity, - std::deque* camera_motions, - std::deque* features); - - // Helper function to subtract current metadata motion from features. Used - // for hybrid estimation case. - void SubtractMetaMotion(const CameraMotion& meta_motion, - RegionFlowFeatureList* features); - - // Inverse of above function to add back meta motion and replace - // feature location with originals after estimation. - void AddMetaMotion(const CameraMotion& meta_motion, - const RegionFlowFeatureList& meta_features, - RegionFlowFeatureList* features, CameraMotion* motion); - - MotionAnalysisCalculatorOptions options_; - int frame_width_ = -1; - int frame_height_ = -1; - int frame_idx_ = 0; - - // Buffers incoming video frame packets (if visualization output is requested) - std::vector packet_buffer_; - - // Buffers incoming timestamps until MotionAnalysis is ready to output via - // above OutputMotionAnalyzedFrames. - std::vector timestamp_buffer_; - - // Input indicators for each stream. - bool selection_input_ = false; - bool video_input_ = false; - - // Output indicators for each stream. - bool region_flow_feature_output_ = false; - bool camera_motion_output_ = false; - bool saliency_output_ = false; - bool visualize_output_ = false; - bool dense_foreground_output_ = false; - bool video_output_ = false; - bool grayscale_output_ = false; - bool csv_file_input_ = false; - - // Inidicates if saliency should be computed. - bool with_saliency_ = false; - - // Set if hybrid meta analysis - see proto for details. - bool hybrid_meta_analysis_ = false; - - // Concatenated motions for each selected frame. Used in case - // hybrid estimation is requested to fallback to valid models. - std::deque selected_motions_; - - // Normalized homographies from CSV file or metadata. - std::deque meta_homographies_; - std::deque meta_motions_; - std::deque meta_features_; - - // Offset into above meta_motions_ and features_ when using - // hybrid meta analysis. - int hybrid_meta_offset_ = 0; - - std::unique_ptr motion_analysis_; - - std::unique_ptr row_weights_; -}; - -REGISTER_CALCULATOR(MotionAnalysisCalculator); - -absl::Status MotionAnalysisCalculator::GetContract(CalculatorContract* cc) { - if (cc->Inputs().HasTag("VIDEO")) { - cc->Inputs().Tag("VIDEO").Set(); - } - - // Optional input stream from frame selection calculator. - if (cc->Inputs().HasTag("SELECTION")) { - cc->Inputs().Tag("SELECTION").Set(); - } - - RET_CHECK(cc->Inputs().HasTag("VIDEO") || cc->Inputs().HasTag("SELECTION")) - << "Either VIDEO, SELECTION must be specified."; - - if (cc->Outputs().HasTag("FLOW")) { - cc->Outputs().Tag("FLOW").Set(); - } - - if (cc->Outputs().HasTag("CAMERA")) { - cc->Outputs().Tag("CAMERA").Set(); - } - - if (cc->Outputs().HasTag("SALIENCY")) { - cc->Outputs().Tag("SALIENCY").Set(); - } - - if (cc->Outputs().HasTag("VIZ")) { - cc->Outputs().Tag("VIZ").Set(); - } - - if (cc->Outputs().HasTag("DENSE_FG")) { - cc->Outputs().Tag("DENSE_FG").Set(); - } - - if (cc->Outputs().HasTag("VIDEO_OUT")) { - cc->Outputs().Tag("VIDEO_OUT").Set(); - } - - if (cc->Outputs().HasTag("GRAY_VIDEO_OUT")) { - // We only output grayscale video if we're actually performing full region- - // flow analysis on the video. - RET_CHECK(cc->Inputs().HasTag("VIDEO") && - !cc->Inputs().HasTag("SELECTION")); - cc->Outputs().Tag("GRAY_VIDEO_OUT").Set(); - } - - if (cc->InputSidePackets().HasTag("CSV_FILE")) { - cc->InputSidePackets().Tag("CSV_FILE").Set(); - } - if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) { - cc->InputSidePackets().Tag("DOWNSAMPLE").Set(); - } - - if (cc->InputSidePackets().HasTag(kOptionsTag)) { - cc->InputSidePackets().Tag(kOptionsTag).Set(); - } - - return absl::OkStatus(); -} - -absl::Status MotionAnalysisCalculator::Open(CalculatorContext* cc) { - options_ = - tool::RetrieveOptions(cc->Options(), - cc->InputSidePackets(), kOptionsTag); - - video_input_ = cc->Inputs().HasTag("VIDEO"); - selection_input_ = cc->Inputs().HasTag("SELECTION"); - region_flow_feature_output_ = cc->Outputs().HasTag("FLOW"); - camera_motion_output_ = cc->Outputs().HasTag("CAMERA"); - saliency_output_ = cc->Outputs().HasTag("SALIENCY"); - visualize_output_ = cc->Outputs().HasTag("VIZ"); - dense_foreground_output_ = cc->Outputs().HasTag("DENSE_FG"); - video_output_ = cc->Outputs().HasTag("VIDEO_OUT"); - grayscale_output_ = cc->Outputs().HasTag("GRAY_VIDEO_OUT"); - csv_file_input_ = cc->InputSidePackets().HasTag("CSV_FILE"); - hybrid_meta_analysis_ = options_.meta_analysis() == - MotionAnalysisCalculatorOptions::META_ANALYSIS_HYBRID; - - if (video_output_) { - RET_CHECK(selection_input_) << "VIDEO_OUT requires SELECTION input"; - } - - if (selection_input_) { - switch (options_.selection_analysis()) { - case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION: - RET_CHECK(!visualize_output_) - << "Visualization not supported for NO_ANALYSIS_USE_SELECTION"; - RET_CHECK(!dense_foreground_output_) - << "Dense foreground not supported for NO_ANALYSIS_USE_SELECTION"; - RET_CHECK(!saliency_output_) - << "Saliency output not supported for NO_ANALYSIS_USE_SELECTION"; - break; - - case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE: - case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED: - RET_CHECK(video_input_) << "Need video input for feature tracking."; - break; - - case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES: - // Nothing to add here. - break; - } - } - - if (visualize_output_ || dense_foreground_output_ || video_output_) { - RET_CHECK(video_input_) << "Video input required."; - } - - if (csv_file_input_) { - RET_CHECK(!selection_input_) - << "Can not use selection input with csv input."; - if (!hybrid_meta_analysis_) { - RET_CHECK(!saliency_output_ && !visualize_output_ && - !dense_foreground_output_ && !grayscale_output_) - << "CSV file and meta input only supports flow and camera motion " - << "output when using metadata only."; - } - } - - if (csv_file_input_) { - // Read from file and parse. - const std::string filename = - cc->InputSidePackets().Tag("CSV_FILE").Get(); - - std::string file_contents; - std::ifstream input_file(filename, std::ios::in); - input_file.seekg(0, std::ios::end); - const int file_length = input_file.tellg(); - file_contents.resize(file_length); - input_file.seekg(0, std::ios::beg); - input_file.read(&file_contents[0], file_length); - input_file.close(); - - RET_CHECK(ParseModelCSV(file_contents, &meta_homographies_)) - << "Could not parse CSV file"; - } - - // Get video header from video or selection input if present. - const VideoHeader* video_header = nullptr; - if (video_input_ && !cc->Inputs().Tag("VIDEO").Header().IsEmpty()) { - video_header = &(cc->Inputs().Tag("VIDEO").Header().Get()); - } else if (selection_input_ && - !cc->Inputs().Tag("SELECTION").Header().IsEmpty()) { - video_header = &(cc->Inputs().Tag("SELECTION").Header().Get()); - } else { - LOG(WARNING) << "No input video header found. Downstream calculators " - "expecting video headers are likely to fail."; - } - - with_saliency_ = options_.analysis_options().compute_motion_saliency(); - // Force computation of saliency if requested as output. - if (cc->Outputs().HasTag("SALIENCY")) { - with_saliency_ = true; - if (!options_.analysis_options().compute_motion_saliency()) { - LOG(WARNING) << "Enable saliency computation. Set " - << "compute_motion_saliency to true to silence this " - << "warning."; - options_.mutable_analysis_options()->set_compute_motion_saliency(true); - } - } - - if (options_.bypass_mode()) { - cc->SetOffset(TimestampDiff(0)); - } - - if (cc->InputSidePackets().HasTag("DOWNSAMPLE")) { - options_.mutable_analysis_options() - ->mutable_flow_options() - ->set_downsample_factor( - cc->InputSidePackets().Tag("DOWNSAMPLE").Get()); - } - - // If no video header is provided, just return and initialize on the first - // Process() call. - if (video_header == nullptr) { - return absl::OkStatus(); - } - - ////////////// EARLY RETURN; ONLY HEADER OUTPUT SHOULD GO HERE /////////////// - - if (visualize_output_) { - cc->Outputs().Tag("VIZ").SetHeader(Adopt(new VideoHeader(*video_header))); - } - - if (video_output_) { - cc->Outputs() - .Tag("VIDEO_OUT") - .SetHeader(Adopt(new VideoHeader(*video_header))); - } - - if (cc->Outputs().HasTag("DENSE_FG")) { - std::unique_ptr foreground_header( - new VideoHeader(*video_header)); - foreground_header->format = ImageFormat::GRAY8; - cc->Outputs().Tag("DENSE_FG").SetHeader(Adopt(foreground_header.release())); - } - - if (cc->Outputs().HasTag("CAMERA")) { - cc->Outputs().Tag("CAMERA").SetHeader( - Adopt(new VideoHeader(*video_header))); - } - - if (cc->Outputs().HasTag("SALIENCY")) { - cc->Outputs() - .Tag("SALIENCY") - .SetHeader(Adopt(new VideoHeader(*video_header))); - } - - return absl::OkStatus(); -} - -absl::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) { - if (options_.bypass_mode()) { - return absl::OkStatus(); - } - - InputStream* video_stream = - video_input_ ? &(cc->Inputs().Tag("VIDEO")) : nullptr; - InputStream* selection_stream = - selection_input_ ? &(cc->Inputs().Tag("SELECTION")) : nullptr; - - // Checked on Open. - CHECK(video_stream || selection_stream); - - // Lazy init. - if (frame_width_ < 0 || frame_height_ < 0) { - MP_RETURN_IF_ERROR(InitOnProcess(video_stream, selection_stream)); - } - - const Timestamp timestamp = cc->InputTimestamp(); - if ((csv_file_input_) && !hybrid_meta_analysis_) { - if (camera_motion_output_) { - RET_CHECK(!meta_motions_.empty()) << "Insufficient metadata."; - - CameraMotion output_motion = meta_motions_.front(); - meta_motions_.pop_front(); - output_motion.set_timestamp_usec(timestamp.Value()); - cc->Outputs().Tag("CAMERA").Add(new CameraMotion(output_motion), - timestamp); - } - - if (region_flow_feature_output_) { - RET_CHECK(!meta_features_.empty()) << "Insufficient frames in CSV file"; - RegionFlowFeatureList output_features = meta_features_.front(); - meta_features_.pop_front(); - - output_features.set_timestamp_usec(timestamp.Value()); - cc->Outputs().Tag("FLOW").Add(new RegionFlowFeatureList(output_features), - timestamp); - } - - ++frame_idx_; - return absl::OkStatus(); - } - - if (motion_analysis_ == nullptr) { - // We do not need MotionAnalysis when using just metadata. - motion_analysis_.reset(new MotionAnalysis(options_.analysis_options(), - frame_width_, frame_height_)); - } - - std::unique_ptr frame_selection_result; - // Always use frame if selection is not activated. - bool use_frame = !selection_input_; - if (selection_input_) { - CHECK(selection_stream); - - // Fill in timestamps we process. - if (!selection_stream->Value().IsEmpty()) { - ASSIGN_OR_RETURN( - frame_selection_result, - selection_stream->Value().ConsumeOrCopy()); - use_frame = true; - - // Make sure both features and camera motion are present. - RET_CHECK(frame_selection_result->has_camera_motion() && - frame_selection_result->has_features()) - << "Frame selection input error at: " << timestamp - << " both camera motion and features need to be " - "present in FrameSelectionResult. " - << frame_selection_result->has_camera_motion() << " , " - << frame_selection_result->has_features(); - } - } - - if (selection_input_ && use_frame && - options_.selection_analysis() == - MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION) { - // Output concatenated results, nothing to compute here. - if (camera_motion_output_) { - cc->Outputs().Tag("CAMERA").Add( - frame_selection_result->release_camera_motion(), timestamp); - } - if (region_flow_feature_output_) { - cc->Outputs().Tag("FLOW").Add(frame_selection_result->release_features(), - timestamp); - } - - if (video_output_) { - cc->Outputs().Tag("VIDEO_OUT").AddPacket(video_stream->Value()); - } - - return absl::OkStatus(); - } - - if (use_frame) { - if (!selection_input_) { - const cv::Mat input_view = - formats::MatView(&video_stream->Get()); - if (hybrid_meta_analysis_) { - // Seed with meta homography. - RET_CHECK(hybrid_meta_offset_ < meta_motions_.size()) - << "Not enough metadata received for hybrid meta analysis"; - Homography initial_transform = - meta_motions_[hybrid_meta_offset_].homography(); - std::function subtract_helper = std::bind( - &MotionAnalysisCalculator::SubtractMetaMotion, this, - meta_motions_[hybrid_meta_offset_], std::placeholders::_1); - - // Keep original features before modification around. - motion_analysis_->AddFrameGeneric( - input_view, timestamp.Value(), initial_transform, nullptr, nullptr, - &subtract_helper, &meta_features_[hybrid_meta_offset_]); - ++hybrid_meta_offset_; - } else { - motion_analysis_->AddFrame(input_view, timestamp.Value()); - } - } else { - selected_motions_.push_back(frame_selection_result->camera_motion()); - switch (options_.selection_analysis()) { - case MotionAnalysisCalculatorOptions::NO_ANALYSIS_USE_SELECTION: - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Should not reach this point!"; - - case MotionAnalysisCalculatorOptions::ANALYSIS_FROM_FEATURES: - motion_analysis_->AddFeatures(frame_selection_result->features()); - break; - - case MotionAnalysisCalculatorOptions::ANALYSIS_RECOMPUTE: { - const cv::Mat input_view = - formats::MatView(&video_stream->Get()); - motion_analysis_->AddFrame(input_view, timestamp.Value()); - break; - } - - case MotionAnalysisCalculatorOptions::ANALYSIS_WITH_SEED: { - Homography homography; - CameraMotionToHomography(frame_selection_result->camera_motion(), - &homography); - const cv::Mat input_view = - formats::MatView(&video_stream->Get()); - motion_analysis_->AddFrameGeneric(input_view, timestamp.Value(), - homography, &homography); - break; - } - } - } - - timestamp_buffer_.push_back(timestamp); - ++frame_idx_; - - VLOG_EVERY_N(0, 100) << "Analyzed frame " << frame_idx_; - - // Buffer input frames only if visualization is requested. - if (visualize_output_ || video_output_) { - packet_buffer_.push_back(video_stream->Value()); - } - - // If requested, output grayscale thumbnails - if (grayscale_output_) { - cv::Mat grayscale_mat = motion_analysis_->GetGrayscaleFrameFromResults(); - std::unique_ptr grayscale_image(new ImageFrame( - ImageFormat::GRAY8, grayscale_mat.cols, grayscale_mat.rows)); - cv::Mat image_frame_mat = formats::MatView(grayscale_image.get()); - grayscale_mat.copyTo(image_frame_mat); - - cc->Outputs() - .Tag("GRAY_VIDEO_OUT") - .Add(grayscale_image.release(), timestamp); - } - - // Output other results, if we have any yet. - OutputMotionAnalyzedFrames(false, cc); - } - - return absl::OkStatus(); -} - -absl::Status MotionAnalysisCalculator::Close(CalculatorContext* cc) { - // Guard against empty videos. - if (motion_analysis_) { - OutputMotionAnalyzedFrames(true, cc); - } - if (csv_file_input_) { - if (!meta_motions_.empty()) { - LOG(ERROR) << "More motions than frames. Unexpected! Remainder: " - << meta_motions_.size(); - } - } - return absl::OkStatus(); -} - -void MotionAnalysisCalculator::OutputMotionAnalyzedFrames( - bool flush, CalculatorContext* cc) { - std::vector> features; - std::vector> camera_motions; - std::vector> saliency; - - const int buffer_size = timestamp_buffer_.size(); - const int num_results = motion_analysis_->GetResults( - flush, &features, &camera_motions, with_saliency_ ? &saliency : nullptr); - - CHECK_LE(num_results, buffer_size); - - if (num_results == 0) { - return; - } - - for (int k = 0; k < num_results; ++k) { - // Region flow features and camera motion for this frame. - auto& feature_list = features[k]; - auto& camera_motion = camera_motions[k]; - const Timestamp timestamp = timestamp_buffer_[k]; - - if (selection_input_ && options_.hybrid_selection_camera()) { - if (camera_motion->type() > selected_motions_.front().type()) { - // Composited type is more stable. - camera_motion->Swap(&selected_motions_.front()); - } - selected_motions_.pop_front(); - } - - if (hybrid_meta_analysis_) { - AddMetaMotion(meta_motions_.front(), meta_features_.front(), - feature_list.get(), camera_motion.get()); - meta_motions_.pop_front(); - meta_features_.pop_front(); - } - - // Video frame for visualization. - std::unique_ptr visualization_frame; - cv::Mat visualization; - if (visualize_output_) { - // Initialize visualization frame with original frame. - visualization_frame.reset(new ImageFrame()); - visualization_frame->CopyFrom(packet_buffer_[k].Get(), 16); - visualization = formats::MatView(visualization_frame.get()); - - motion_analysis_->RenderResults( - *feature_list, *camera_motion, - with_saliency_ ? saliency[k].get() : nullptr, &visualization); - - cc->Outputs().Tag("VIZ").Add(visualization_frame.release(), timestamp); - } - - // Output dense foreground mask. - if (dense_foreground_output_) { - std::unique_ptr foreground_frame( - new ImageFrame(ImageFormat::GRAY8, frame_width_, frame_height_)); - cv::Mat foreground = formats::MatView(foreground_frame.get()); - motion_analysis_->ComputeDenseForeground(*feature_list, *camera_motion, - &foreground); - cc->Outputs().Tag("DENSE_FG").Add(foreground_frame.release(), timestamp); - } - - // Output flow features if requested. - if (region_flow_feature_output_) { - cc->Outputs().Tag("FLOW").Add(feature_list.release(), timestamp); - } - - // Output camera motion. - if (camera_motion_output_) { - cc->Outputs().Tag("CAMERA").Add(camera_motion.release(), timestamp); - } - - if (video_output_) { - cc->Outputs().Tag("VIDEO_OUT").AddPacket(packet_buffer_[k]); - } - - // Output saliency. - if (saliency_output_) { - cc->Outputs().Tag("SALIENCY").Add(saliency[k].release(), timestamp); - } - } - - if (hybrid_meta_analysis_) { - hybrid_meta_offset_ -= num_results; - CHECK_GE(hybrid_meta_offset_, 0); - } - - timestamp_buffer_.erase(timestamp_buffer_.begin(), - timestamp_buffer_.begin() + num_results); - - if (visualize_output_ || video_output_) { - packet_buffer_.erase(packet_buffer_.begin(), - packet_buffer_.begin() + num_results); - } -} - -absl::Status MotionAnalysisCalculator::InitOnProcess( - InputStream* video_stream, InputStream* selection_stream) { - if (video_stream) { - frame_width_ = video_stream->Get().Width(); - frame_height_ = video_stream->Get().Height(); - - // Ensure image options are set correctly. - auto* region_options = - options_.mutable_analysis_options()->mutable_flow_options(); - - // Use two possible formats to account for different channel orders. - RegionFlowComputationOptions::ImageFormat image_format; - RegionFlowComputationOptions::ImageFormat image_format2; - switch (video_stream->Get().Format()) { - case ImageFormat::GRAY8: - image_format = image_format2 = - RegionFlowComputationOptions::FORMAT_GRAYSCALE; - break; - - case ImageFormat::SRGB: - image_format = RegionFlowComputationOptions::FORMAT_RGB; - image_format2 = RegionFlowComputationOptions::FORMAT_BGR; - break; - - case ImageFormat::SRGBA: - image_format = RegionFlowComputationOptions::FORMAT_RGBA; - image_format2 = RegionFlowComputationOptions::FORMAT_BGRA; - break; - - default: - RET_CHECK(false) << "Unsupported image format."; - } - if (region_options->image_format() != image_format && - region_options->image_format() != image_format2) { - LOG(WARNING) << "Requested image format in RegionFlowComputation " - << "does not match video stream format. Overriding."; - region_options->set_image_format(image_format); - } - - // Account for downsampling mode INPUT_SIZE. In this case we are handed - // already downsampled frames but the resulting CameraMotion should - // be computed on higher resolution as specifed by the downsample scale. - if (region_options->downsample_mode() == - RegionFlowComputationOptions::DOWNSAMPLE_TO_INPUT_SIZE) { - const float scale = region_options->downsample_factor(); - frame_width_ = static_cast(std::round(frame_width_ * scale)); - frame_height_ = static_cast(std::round(frame_height_ * scale)); - } - } else if (selection_stream) { - const auto& camera_motion = - selection_stream->Get().camera_motion(); - frame_width_ = camera_motion.frame_width(); - frame_height_ = camera_motion.frame_height(); - } else { - LOG(FATAL) << "Either VIDEO or SELECTION stream need to be specified."; - } - - // Filled by CSV file parsing. - if (!meta_homographies_.empty()) { - CHECK(csv_file_input_); - AppendCameraMotionsFromHomographies(meta_homographies_, - true, // append identity. - &meta_motions_, &meta_features_); - meta_homographies_.clear(); - } - - // Filter weights before using for hybrid mode. - if (hybrid_meta_analysis_) { - auto* motion_options = - options_.mutable_analysis_options()->mutable_motion_options(); - motion_options->set_filter_initialized_irls_weights(true); - } - - return absl::OkStatus(); -} - -bool MotionAnalysisCalculator::ParseModelCSV( - const std::string& contents, std::deque* homographies) { - std::vector values = - absl::StrSplit(contents, absl::ByAnyChar(",\n")); - - // Trim off any empty lines. - while (values.back().empty()) { - values.pop_back(); - } - - // Convert to float. - std::vector homog_values; - homog_values.reserve(values.size()); - - for (const auto& value : values) { - double value_64f; - if (!absl::SimpleAtod(value, &value_64f)) { - LOG(ERROR) << "Not a double, expected!"; - return false; - } - - homog_values.push_back(value_64f); - } - - return HomographiesFromValues(homog_values, homographies); -} - -bool MotionAnalysisCalculator::HomographiesFromValues( - const std::vector& homog_values, - std::deque* homographies) { - CHECK(homographies); - - // Obvious constants are obvious :D - constexpr int kHomographyValues = 9; - if (homog_values.size() % kHomographyValues != 0) { - LOG(ERROR) << "Contents not a multiple of " << kHomographyValues; - return false; - } - - for (int k = 0; k < homog_values.size(); k += kHomographyValues) { - std::vector h_vals(kHomographyValues); - for (int l = 0; l < kHomographyValues; ++l) { - h_vals[l] = homog_values[k + l]; - } - - // Normalize last entry to 1. - if (h_vals[kHomographyValues - 1] == 0) { - LOG(ERROR) << "Degenerate homography, last entry is zero"; - return false; - } - - const double scale = 1.0f / h_vals[kHomographyValues - 1]; - for (int l = 0; l < kHomographyValues; ++l) { - h_vals[l] *= scale; - } - - Homography h = HomographyAdapter::FromDoublePointer(h_vals.data(), false); - homographies->push_back(h); - } - - if (homographies->size() % options_.meta_models_per_frame() != 0) { - LOG(ERROR) << "Total homographies not a multiple of specified models " - << "per frame."; - return false; - } - - return true; -} - -void MotionAnalysisCalculator::SubtractMetaMotion( - const CameraMotion& meta_motion, RegionFlowFeatureList* features) { - if (meta_motion.mixture_homography().model_size() > 0) { - CHECK(row_weights_ != nullptr); - RegionFlowFeatureListViaTransform(meta_motion.mixture_homography(), - features, -1.0f, - 1.0f, // subtract transformed. - true, // replace feature loc. - row_weights_.get()); - } else { - RegionFlowFeatureListViaTransform(meta_motion.homography(), features, -1.0f, - 1.0f, // subtract transformed. - true); // replace feature loc. - } - - // Clamp transformed features to domain and handle outliers. - const float domain_diam = - hypot(features->frame_width(), features->frame_height()); - const float motion_mag = meta_motion.average_magnitude(); - // Same irls fraction as used by MODEL_MIXTURE_HOMOGRAPHY scaling in - // MotionEstimation. - const float irls_fraction = options_.analysis_options() - .motion_options() - .irls_mixture_fraction_scale() * - options_.analysis_options() - .motion_options() - .irls_motion_magnitude_fraction(); - float err_scale = std::max(1.0f, motion_mag * irls_fraction); - - const float max_err = - options_.meta_outlier_domain_ratio() * domain_diam * err_scale; - const float max_err_sq = max_err * max_err; - - for (auto& feature : *features->mutable_feature()) { - feature.set_x( - std::max(0.0f, std::min(features->frame_width() - 1.0f, feature.x()))); - feature.set_y( - std::max(0.0f, std::min(features->frame_height() - 1.0f, feature.y()))); - // Label anything with large residual motion an outlier. - if (FeatureFlow(feature).Norm2() > max_err_sq) { - feature.set_irls_weight(0.0f); - } - } -} - -void MotionAnalysisCalculator::AddMetaMotion( - const CameraMotion& meta_motion, const RegionFlowFeatureList& meta_features, - RegionFlowFeatureList* features, CameraMotion* motion) { - // Restore old feature location. - CHECK_EQ(meta_features.feature_size(), features->feature_size()); - for (int k = 0; k < meta_features.feature_size(); ++k) { - auto feature = features->mutable_feature(k); - const auto& meta_feature = meta_features.feature(k); - feature->set_x(meta_feature.x()); - feature->set_y(meta_feature.y()); - feature->set_dx(meta_feature.dx()); - feature->set_dy(meta_feature.dy()); - } - - // Composite camera motion. - *motion = ComposeCameraMotion(*motion, meta_motion); - // Restore type from metadata, i.e. do not declare motions as invalid. - motion->set_type(meta_motion.type()); - motion->set_match_frame(-1); -} - -void MotionAnalysisCalculator::AppendCameraMotionsFromHomographies( - const std::deque& homographies, bool append_identity, - std::deque* camera_motions, - std::deque* features) { - CHECK(camera_motions); - CHECK(features); - - CameraMotion identity; - identity.set_frame_width(frame_width_); - identity.set_frame_height(frame_height_); - - *identity.mutable_translation() = TranslationModel(); - *identity.mutable_linear_similarity() = LinearSimilarityModel(); - *identity.mutable_homography() = Homography(); - identity.set_type(CameraMotion::VALID); - identity.set_match_frame(0); - - RegionFlowFeatureList empty_list; - empty_list.set_long_tracks(true); - empty_list.set_match_frame(-1); - empty_list.set_frame_width(frame_width_); - empty_list.set_frame_height(frame_height_); - - if (append_identity) { - camera_motions->push_back(identity); - features->push_back(empty_list); - } - - const int models_per_frame = options_.meta_models_per_frame(); - CHECK_GT(models_per_frame, 0) << "At least one model per frame is needed"; - CHECK_EQ(0, homographies.size() % models_per_frame); - const int num_frames = homographies.size() / models_per_frame; - - // Heuristic sigma, similar to what we use for rolling shutter removal. - const float mixture_sigma = 1.0f / models_per_frame; - - if (row_weights_ == nullptr) { - row_weights_.reset(new MixtureRowWeights(frame_height_, - frame_height_ / 10, // 10% margin - mixture_sigma * frame_height_, - 1.0f, models_per_frame)); - } - - for (int f = 0; f < num_frames; ++f) { - MixtureHomography mix_homog; - const int model_start = f * models_per_frame; - - for (int k = 0; k < models_per_frame; ++k) { - const Homography& homog = homographies[model_start + k]; - *mix_homog.add_model() = ModelInvert(homog); - } - - CameraMotion c = identity; - c.set_match_frame(-1); - - if (mix_homog.model_size() > 1) { - *c.mutable_mixture_homography() = mix_homog; - c.set_mixture_row_sigma(mixture_sigma); - - for (int k = 0; k < models_per_frame; ++k) { - c.add_mixture_inlier_coverage(1.0f); - } - *c.add_mixture_homography_spectrum() = mix_homog; - c.set_rolling_shutter_motion_index(0); - - *c.mutable_homography() = ProjectViaFit( - mix_homog, frame_width_, frame_height_, row_weights_.get()); - } else { - // Guaranteed to exist because to check that models_per_frame > 0 above. - *c.mutable_homography() = mix_homog.model(0); - } - - // Project remaining motions down. - *c.mutable_linear_similarity() = ProjectViaFit( - c.homography(), frame_width_, frame_height_); - *c.mutable_translation() = ProjectViaFit( - c.homography(), frame_width_, frame_height_); - - c.set_average_magnitude( - std::hypot(c.translation().dx(), c.translation().dy())); - - camera_motions->push_back(c); - features->push_back(empty_list); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/motion_analysis_calculator.proto b/mediapipe/calculators/video/motion_analysis_calculator.proto deleted file mode 100644 index a2af77d12..000000000 --- a/mediapipe/calculators/video/motion_analysis_calculator.proto +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/tracking/motion_analysis.proto"; - -// Next tag: 10 -message MotionAnalysisCalculatorOptions { - extend CalculatorOptions { - optional MotionAnalysisCalculatorOptions ext = 270698255; - } - - optional mediapipe.MotionAnalysisOptions analysis_options = 1; - - // Determines how optional input SELECTION (if present) is used to compute - // the final camera motion. - enum SelectionAnalysis { - // Recompute camera motion for selected frame neighbors. - ANALYSIS_RECOMPUTE = 1; - - // Use composited camera motion and region flow from SELECTION input. No - // tracking or re-computation is performed. - // Note that in this case only CAMERA, FLOW and VIDEO_OUT tags are - // supported as output. - NO_ANALYSIS_USE_SELECTION = 2; - - // Recompute camera motion for selected frame neighbors using - // features supplied by SELECTION input. No feature tracking is performed. - ANALYSIS_FROM_FEATURES = 3; - - // Recomputes camera motion for selected frame neighbors but seeds - // initial transform with camera motion from SELECTION input. - ANALYSIS_WITH_SEED = 4; - } - - optional SelectionAnalysis selection_analysis = 4 - [default = ANALYSIS_WITH_SEED]; - - // If activated when SELECTION input is activated, will replace the computed - // camera motion (for any of the ANALYSIS_* case above) with the one supplied - // by the frame selection, in case the frame selection one is more stable. - // For example, if recomputed camera motion is unstable but the one from - // the selection result is stable, will use the stable result instead. - optional bool hybrid_selection_camera = 5 [default = false]; - - // Determines how optional input META is used to compute the final camera - // motion. - enum MetaAnalysis { - // Uses metadata supplied motions as is. - META_ANALYSIS_USE_META = 1; - - // Seeds visual tracking from metadata motions - estimates visual residual - // motion and combines with metadata. - META_ANALYSIS_HYBRID = 2; - } - - optional MetaAnalysis meta_analysis = 8 [default = META_ANALYSIS_USE_META]; - - // Determines number of homography models per frame stored in the CSV file - // or the homography metadata in META. - // For values > 1, MixtureHomographies are created. - optional int32 meta_models_per_frame = 6 [default = 1]; - - // Used for META_ANALYSIS_HYBRID. Rejects features which flow deviates - // domain_ratio * image diagonal size from the ground truth metadata motion. - optional float meta_outlier_domain_ratio = 9 [default = 0.0015]; - - // If true, the MotionAnalysisCalculator will skip all processing and emit no - // packets on any output. This is useful for quickly creating different - // versions of a MediaPipe graph without changing its structure, assuming that - // downstream calculators can handle missing input packets. - // TODO: Remove this hack. See b/36485206 for more details. - optional bool bypass_mode = 7 [default = false]; -} - -// Taken from -// java/com/google/android/libraries/microvideo/proto/microvideo.proto to -// satisfy leakr requirements -// TODO: Remove and use above proto. -message HomographyData { - // For each frame, there are 12 homography matrices stored. Each matrix is - // 3x3 (9 elements). This field will contain 12 x 3 x 3 float values. The - // first row of the first homography matrix will be followed by the second row - // of the first homography matrix, followed by third row of first homography - // matrix, followed by the first row of the second homography matrix, etc. - repeated float motion_homography_data = 1 [packed = true]; - - // Vector containing histogram counts for individual patches in the frame. - repeated uint32 histogram_count_data = 2 [packed = true]; - - // The width of the frame at the time metadata was sampled. - optional int32 frame_width = 3; - - // The height of the frame at the time metadata was sampled. - optional int32 frame_height = 4; -} diff --git a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc deleted file mode 100644 index bf7ed3e8a..000000000 --- a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/tool/status_util.h" - -namespace mediapipe { - -namespace { -// cv::VideoCapture set data type to unsigned char by default. Therefore, the -// image format is only related to the number of channles the cv::Mat has. -ImageFormat::Format GetImageFormat(int num_channels) { - ImageFormat::Format format; - switch (num_channels) { - case 1: - format = ImageFormat::GRAY8; - break; - case 3: - format = ImageFormat::SRGB; - break; - case 4: - format = ImageFormat::SRGBA; - break; - default: - format = ImageFormat::UNKNOWN; - break; - } - return format; -} -} // namespace - -// This Calculator takes no input streams and produces video packets. -// All streams and input side packets are specified using tags and all of them -// are optional. -// -// Output Streams: -// VIDEO: Output video frames (ImageFrame). -// VIDEO_PRESTREAM: -// Optional video header information output at -// Timestamp::PreStream() for the corresponding stream. -// Input Side Packets: -// INPUT_FILE_PATH: The input file path. -// -// Example config: -// node { -// calculator: "OpenCvVideoDecoderCalculator" -// input_side_packet: "INPUT_FILE_PATH:input_file_path" -// output_stream: "VIDEO:video_frames" -// output_stream: "VIDEO_PRESTREAM:video_header" -// } -// -// OpenCV's VideoCapture doesn't decode audio tracks. If the audio tracks need -// to be saved, specify an output side packet with tag "SAVED_AUDIO_PATH". -// The calculator will call FFmpeg binary to save audio tracks as an aac file. -// If the audio tracks can't be extracted by FFmpeg, the output side packet -// will contain an empty std::string. -// -// Example config: -// node { -// calculator: "OpenCvVideoDecoderCalculator" -// input_side_packet: "INPUT_FILE_PATH:input_file_path" -// output_side_packet: "SAVED_AUDIO_PATH:audio_path" -// output_stream: "VIDEO:video_frames" -// output_stream: "VIDEO_PRESTREAM:video_header" -// } -// -class OpenCvVideoDecoderCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set(); - cc->Outputs().Tag("VIDEO").Set(); - if (cc->Outputs().HasTag("VIDEO_PRESTREAM")) { - cc->Outputs().Tag("VIDEO_PRESTREAM").Set(); - } - if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) { - cc->OutputSidePackets().Tag("SAVED_AUDIO_PATH").Set(); - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const std::string& input_file_path = - cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get(); - cap_ = absl::make_unique(input_file_path); - if (!cap_->isOpened()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Fail to open video file at " << input_file_path; - } - width_ = static_cast(cap_->get(cv::CAP_PROP_FRAME_WIDTH)); - height_ = static_cast(cap_->get(cv::CAP_PROP_FRAME_HEIGHT)); - double fps = static_cast(cap_->get(cv::CAP_PROP_FPS)); - frame_count_ = static_cast(cap_->get(cv::CAP_PROP_FRAME_COUNT)); - // Unfortunately, cap_->get(cv::CAP_PROP_FORMAT) always returns CV_8UC1 - // back. To get correct image format, we read the first frame from the video - // and get the number of channels. - cv::Mat frame; - cap_->read(frame); - if (frame.empty()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Fail to read any frames from the video file at " - << input_file_path; - } - format_ = GetImageFormat(frame.channels()); - if (format_ == ImageFormat::UNKNOWN) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported video format of the video file at " - << input_file_path; - } - - if (fps <= 0 || frame_count_ <= 0 || width_ <= 0 || height_ <= 0) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Fail to make video header due to the incorrect metadata from " - "the video file at " - << input_file_path; - } - auto header = absl::make_unique(); - header->format = format_; - header->width = width_; - header->height = height_; - header->frame_rate = fps; - header->duration = frame_count_ / fps; - - if (cc->Outputs().HasTag("VIDEO_PRESTREAM")) { - cc->Outputs() - .Tag("VIDEO_PRESTREAM") - .Add(header.release(), Timestamp::PreStream()); - cc->Outputs().Tag("VIDEO_PRESTREAM").Close(); - } - // Rewind to the very first frame. - cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0); - - if (cc->OutputSidePackets().HasTag("SAVED_AUDIO_PATH")) { -#ifdef HAVE_FFMPEG - std::string saved_audio_path = std::tmpnam(nullptr); - std::string ffmpeg_command = - absl::StrCat("ffmpeg -nostats -loglevel 0 -i ", input_file_path, - " -vn -f adts ", saved_audio_path); - system(ffmpeg_command.c_str()); - int status_code = system(absl::StrCat("ls ", saved_audio_path).c_str()); - if (status_code == 0) { - cc->OutputSidePackets() - .Tag("SAVED_AUDIO_PATH") - .Set(MakePacket(saved_audio_path)); - } else { - LOG(WARNING) << "FFmpeg can't extract audio from " << input_file_path - << " by executing the following command: " - << ffmpeg_command; - cc->OutputSidePackets() - .Tag("SAVED_AUDIO_PATH") - .Set(MakePacket(std::string())); - } -#else - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "OpenCVVideoDecoderCalculator can't save the audio file " - "because FFmpeg is not installed. Please remove " - "output_side_packet: \"SAVED_AUDIO_PATH\" from the node " - "config."; -#endif - } - return absl::OkStatus(); - } - - absl::Status Process(CalculatorContext* cc) override { - auto image_frame = absl::make_unique(format_, width_, height_, - /*alignment_boundary=*/1); - // Use microsecond as the unit of time. - Timestamp timestamp(cap_->get(cv::CAP_PROP_POS_MSEC) * 1000); - if (format_ == ImageFormat::GRAY8) { - cv::Mat frame = formats::MatView(image_frame.get()); - cap_->read(frame); - if (frame.empty()) { - return tool::StatusStop(); - } - } else { - cv::Mat tmp_frame; - cap_->read(tmp_frame); - if (tmp_frame.empty()) { - return tool::StatusStop(); - } - if (format_ == ImageFormat::SRGB) { - cv::cvtColor(tmp_frame, formats::MatView(image_frame.get()), - cv::COLOR_BGR2RGB); - } else if (format_ == ImageFormat::SRGBA) { - cv::cvtColor(tmp_frame, formats::MatView(image_frame.get()), - cv::COLOR_BGRA2RGBA); - } - } - // If the timestamp of the current frame is not greater than the one of the - // previous frame, the new frame will be discarded. - if (prev_timestamp_ < timestamp) { - cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp); - prev_timestamp_ = timestamp; - decoded_frames_++; - } - - return absl::OkStatus(); - } - - absl::Status Close(CalculatorContext* cc) override { - if (cap_ && cap_->isOpened()) { - cap_->release(); - } - if (decoded_frames_ != frame_count_) { - LOG(WARNING) << "Not all the frames are decoded (total frames: " - << frame_count_ << " vs decoded frames: " << decoded_frames_ - << ")."; - } - return absl::OkStatus(); - } - - private: - std::unique_ptr cap_; - int width_; - int height_; - int frame_count_; - int decoded_frames_ = 0; - ImageFormat::Format format_; - Timestamp prev_timestamp_ = Timestamp::Unset(); -}; - -REGISTER_CALCULATOR(OpenCvVideoDecoderCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc b/mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc deleted file mode 100644 index 03d27b6fe..000000000 --- a/mediapipe/calculators/video/opencv_video_decoder_calculator_test.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { - -TEST(OpenCvVideoDecoderCalculatorTest, TestMp4Avc720pVideo) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream")pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_MP4_AVC720P_AAC.video")); - MP_EXPECT_OK(runner.Run()); - - EXPECT_EQ(runner.Outputs().Tag("VIDEO_PRESTREAM").packets.size(), 1); - MP_EXPECT_OK(runner.Outputs() - .Tag("VIDEO_PRESTREAM") - .packets[0] - .ValidateAsType()); - const mediapipe::VideoHeader& header = - runner.Outputs().Tag("VIDEO_PRESTREAM").packets[0].Get(); - EXPECT_EQ(ImageFormat::SRGB, header.format); - EXPECT_EQ(1280, header.width); - EXPECT_EQ(640, header.height); - EXPECT_FLOAT_EQ(6.0f, header.duration); - EXPECT_FLOAT_EQ(30.0f, header.frame_rate); - // The number of the output packets should be 180. - // Some OpenCV version returns the first two frames with the same timestamp on - // macos and we might miss one frame here. - int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size(); - EXPECT_GE(num_of_packets, 179); - for (int i = 0; i < num_of_packets; ++i) { - Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i]; - cv::Mat output_mat = - formats::MatView(&(image_frame_packet.Get())); - EXPECT_EQ(1280, output_mat.size().width); - EXPECT_EQ(640, output_mat.size().height); - EXPECT_EQ(3, output_mat.channels()); - cv::Scalar s = cv::mean(output_mat); - for (int i = 0; i < 3; ++i) { - EXPECT_GT(s[i], 0); - EXPECT_LT(s[i], 255); - } - } -} - -TEST(OpenCvVideoDecoderCalculatorTest, TestFlvH264Video) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream")pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_FLV_H264_AAC.video")); - MP_EXPECT_OK(runner.Run()); - - EXPECT_EQ(runner.Outputs().Tag("VIDEO_PRESTREAM").packets.size(), 1); - MP_EXPECT_OK(runner.Outputs() - .Tag("VIDEO_PRESTREAM") - .packets[0] - .ValidateAsType()); - const mediapipe::VideoHeader& header = - runner.Outputs().Tag("VIDEO_PRESTREAM").packets[0].Get(); - EXPECT_EQ(ImageFormat::SRGB, header.format); - EXPECT_EQ(640, header.width); - EXPECT_EQ(320, header.height); - // TODO: The actual header.duration is 6.0666666f and the frame_rate - // can be either 30.30303f (with opencv2) or 30f (with opencv3 and opencv4). - // EXPECT_FLOAT_EQ(6.0f, header.duration); - // EXPECT_FLOAT_EQ(30.0f, header.frame_rate); - EXPECT_EQ(180, runner.Outputs().Tag("VIDEO").packets.size()); - for (int i = 0; i < 180; ++i) { - Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i]; - cv::Mat output_mat = - formats::MatView(&(image_frame_packet.Get())); - EXPECT_EQ(640, output_mat.size().width); - EXPECT_EQ(320, output_mat.size().height); - EXPECT_EQ(3, output_mat.channels()); - cv::Scalar s = cv::mean(output_mat); - for (int i = 0; i < 3; ++i) { - EXPECT_GT(s[i], 0); - EXPECT_LT(s[i], 255); - } - } -} - -TEST(OpenCvVideoDecoderCalculatorTest, TestMkvVp8Video) { - CalculatorGraphConfig::Node node_config = - ParseTextProtoOrDie(R"pb( - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream")pb"); - CalculatorRunner runner(node_config); - runner.MutableSidePackets()->Tag("INPUT_FILE_PATH") = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_MKV_VP8_VORBIS.video")); - MP_EXPECT_OK(runner.Run()); - - EXPECT_EQ(runner.Outputs().Tag("VIDEO_PRESTREAM").packets.size(), 1); - MP_EXPECT_OK(runner.Outputs() - .Tag("VIDEO_PRESTREAM") - .packets[0] - .ValidateAsType()); - const mediapipe::VideoHeader& header = - runner.Outputs().Tag("VIDEO_PRESTREAM").packets[0].Get(); - EXPECT_EQ(ImageFormat::SRGB, header.format); - EXPECT_EQ(640, header.width); - EXPECT_EQ(320, header.height); - EXPECT_FLOAT_EQ(6.0f, header.duration); - EXPECT_FLOAT_EQ(30.0f, header.frame_rate); - // The number of the output packets should be 180. - // Some OpenCV version returns the first two frames with the same timestamp on - // macos and we might miss one frame here. - int num_of_packets = runner.Outputs().Tag("VIDEO").packets.size(); - EXPECT_GE(num_of_packets, 179); - for (int i = 0; i < num_of_packets; ++i) { - Packet image_frame_packet = runner.Outputs().Tag("VIDEO").packets[i]; - cv::Mat output_mat = - formats::MatView(&(image_frame_packet.Get())); - EXPECT_EQ(640, output_mat.size().width); - EXPECT_EQ(320, output_mat.size().height); - EXPECT_EQ(3, output_mat.channels()); - cv::Scalar s = cv::mean(output_mat); - for (int i = 0; i < 3; ++i) { - EXPECT_GT(s[i], 0); - EXPECT_LT(s[i], 255); - } - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/video/opencv_video_encoder_calculator.cc b/mediapipe/calculators/video/opencv_video_encoder_calculator.cc deleted file mode 100644 index 9a74fb710..000000000 --- a/mediapipe/calculators/video/opencv_video_encoder_calculator.cc +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include - -#include "absl/strings/str_split.h" -#include "mediapipe/calculators/video/opencv_video_encoder_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/source_location.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" -#include "mediapipe/framework/tool/status_util.h" - -namespace mediapipe { - -// Encodes the input video stream and produces a media file. -// The media file can be output to the output_file_path specified as a side -// packet. Currently, the calculator only supports one video stream (in -// mediapipe::ImageFrame). -// -// Example config: -// node { -// calculator: "OpenCvVideoEncoderCalculator" -// input_stream: "VIDEO:video" -// input_stream: "VIDEO_PRESTREAM:video_header" -// input_side_packet: "OUTPUT_FILE_PATH:output_file_path" -// node_options { -// [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { -// codec: "avc1" -// video_format: "mp4" -// } -// } -// } -// -// OpenCV's VideoWriter doesn't encode audio. If an input side packet with tag -// "AUDIO_FILE_PATH" is specified, the calculator will call FFmpeg binary to -// attach the audio file to the video as the last step in Close(). -// -// Example config: -// node { -// calculator: "OpenCvVideoEncoderCalculator" -// input_stream: "VIDEO:video" -// input_stream: "VIDEO_PRESTREAM:video_header" -// input_side_packet: "OUTPUT_FILE_PATH:output_file_path" -// input_side_packet: "AUDIO_FILE_PATH:audio_path" -// node_options { -// [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { -// codec: "avc1" -// video_format: "mp4" -// } -// } -// } -// -class OpenCvVideoEncoderCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - absl::Status Close(CalculatorContext* cc) override; - - private: - absl::Status SetUpVideoWriter(float frame_rate, int width, int height); - - std::string output_file_path_; - int four_cc_; - std::unique_ptr writer_; -}; - -absl::Status OpenCvVideoEncoderCalculator::GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag("VIDEO")); - cc->Inputs().Tag("VIDEO").Set(); - if (cc->Inputs().HasTag("VIDEO_PRESTREAM")) { - cc->Inputs().Tag("VIDEO_PRESTREAM").Set(); - } - RET_CHECK(cc->InputSidePackets().HasTag("OUTPUT_FILE_PATH")); - cc->InputSidePackets().Tag("OUTPUT_FILE_PATH").Set(); - if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) { - cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Set(); - } - return absl::OkStatus(); -} - -absl::Status OpenCvVideoEncoderCalculator::Open(CalculatorContext* cc) { - OpenCvVideoEncoderCalculatorOptions options = - cc->Options(); - RET_CHECK(options.has_codec() && options.codec().length() == 4) - << "A 4-character codec code must be specified in " - "OpenCvVideoEncoderCalculatorOptions"; - const char* codec_array = options.codec().c_str(); - four_cc_ = mediapipe::fourcc(codec_array[0], codec_array[1], codec_array[2], - codec_array[3]); - RET_CHECK(!options.video_format().empty()) - << "Video format must be specified in " - "OpenCvVideoEncoderCalculatorOptions"; - output_file_path_ = - cc->InputSidePackets().Tag("OUTPUT_FILE_PATH").Get(); - std::vector splited_file_path = - absl::StrSplit(output_file_path_, '.'); - RET_CHECK(splited_file_path.size() >= 2 && - splited_file_path[splited_file_path.size() - 1] == - options.video_format()) - << "The output file path is invalid."; - // If the video header will be available, the video metadata will be fetched - // from the video header directly. The calculator will receive the video - // header packet at timestamp prestream. - if (cc->Inputs().HasTag("VIDEO_PRESTREAM")) { - return absl::OkStatus(); - } - return SetUpVideoWriter(options.fps(), options.width(), options.height()); -} - -absl::Status OpenCvVideoEncoderCalculator::Process(CalculatorContext* cc) { - if (cc->InputTimestamp() == Timestamp::PreStream()) { - const VideoHeader& video_header = - cc->Inputs().Tag("VIDEO_PRESTREAM").Get(); - return SetUpVideoWriter(video_header.frame_rate, video_header.width, - video_header.height); - } - - const ImageFrame& image_frame = - cc->Inputs().Tag("VIDEO").Value().Get(); - ImageFormat::Format format = image_frame.Format(); - cv::Mat frame; - if (format == ImageFormat::GRAY8) { - frame = formats::MatView(&image_frame); - if (frame.empty()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Receive empty frame at timestamp " - << cc->Inputs().Tag("VIDEO").Value().Timestamp() - << " in OpenCvVideoEncoderCalculator::Process()"; - } - } else { - cv::Mat tmp_frame = formats::MatView(&image_frame); - if (tmp_frame.empty()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Receive empty frame at timestamp " - << cc->Inputs().Tag("VIDEO").Value().Timestamp() - << " in OpenCvVideoEncoderCalculator::Process()"; - } - if (format == ImageFormat::SRGB) { - cv::cvtColor(tmp_frame, frame, cv::COLOR_RGB2BGR); - } else if (format == ImageFormat::SRGBA) { - cv::cvtColor(tmp_frame, frame, cv::COLOR_RGBA2BGR); - } else { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported image format: " << format; - } - } - writer_->write(frame); - return absl::OkStatus(); -} - -absl::Status OpenCvVideoEncoderCalculator::Close(CalculatorContext* cc) { - if (writer_ && writer_->isOpened()) { - writer_->release(); - } - if (cc->InputSidePackets().HasTag("AUDIO_FILE_PATH")) { -#ifdef HAVE_FFMPEG - const std::string& audio_file_path = - cc->InputSidePackets().Tag("AUDIO_FILE_PATH").Get(); - if (audio_file_path.empty()) { - LOG(WARNING) << "OpenCvVideoEncoderCalculator isn't able to attach the " - "audio tracks to the generated video because the audio " - "file path is not specified."; - } else { - // A temp output file is needed because FFmpeg can't do in-place editing. - const std::string temp_file_path = std::tmpnam(nullptr); - system(absl::StrCat("mv ", output_file_path_, " ", temp_file_path, - "&& ffmpeg -nostats -loglevel 0 -i ", temp_file_path, - " -i ", audio_file_path, - " -c copy -map 0:v:0 -map 1:a:0 ", output_file_path_, - "&& rm ", temp_file_path) - .c_str()); - } - -#else - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "OpenCVVideoEncoderCalculator can't attach the audio tracks to " - "the video because FFmpeg is not installed. Please remove " - "input_side_packet: \"AUDIO_FILE_PATH\" from the node " - "config."; -#endif - } - return absl::OkStatus(); -} - -absl::Status OpenCvVideoEncoderCalculator::SetUpVideoWriter(float frame_rate, - int width, - int height) { - RET_CHECK(frame_rate > 0 && width > 0 && height > 0) - << "Invalid video metadata: frame_rate=" << frame_rate - << ", width=" << width << ", height=" << height; - writer_ = absl::make_unique( - output_file_path_, four_cc_, frame_rate, cv::Size(width, height)); - if (!writer_->isOpened()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Fail to open file at " << output_file_path_; - } - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(OpenCvVideoEncoderCalculator); -} // namespace mediapipe diff --git a/mediapipe/calculators/video/opencv_video_encoder_calculator.proto b/mediapipe/calculators/video/opencv_video_encoder_calculator.proto deleted file mode 100644 index a3a7af3ac..000000000 --- a/mediapipe/calculators/video/opencv_video_encoder_calculator.proto +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message OpenCvVideoEncoderCalculatorOptions { - extend CalculatorOptions { - optional OpenCvVideoEncoderCalculatorOptions ext = 207936763; - } - // The 4-character code of the codec to encode the video. - optional string codec = 1; - - // The video format of the output video file. - optional string video_format = 2; - - // The frame rate in Hz at which the video frames are output. - optional double fps = 3; - - // Dimensions of the video in pixels. - optional int32 width = 4; - optional int32 height = 5; -} diff --git a/mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc b/mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc deleted file mode 100644 index 7e946d82d..000000000 --- a/mediapipe/calculators/video/opencv_video_encoder_calculator_test.cc +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/deleting_file.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { -// Temporarily disable the test. -// TODO: Investigate the “Could not open codec 'libx264'” error with -// opencv2. -TEST(OpenCvVideoEncoderCalculatorTest, DISABLED_TestMp4Avc720pVideo) { - CalculatorGraphConfig config = - ParseTextProtoOrDie(R"pb( - node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream" - } - node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:video" - input_stream: "VIDEO_PRESTREAM:video_prestream" - input_side_packet: "OUTPUT_FILE_PATH:output_file_path" - node_options { - [type.googleapis.com/ - mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "avc1" - video_format: "mp4" - } - } - } - )pb"); - std::map input_side_packets; - input_side_packets["input_file_path"] = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_MP4_AVC720P_AAC.video")); - const std::string output_file_path = "/tmp/tmp_video.mp4"; - DeletingFile deleting_file(output_file_path, true); - input_side_packets["output_file_path"] = - MakePacket(output_file_path); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config, input_side_packets)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("video_prestream"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - Packet packet; - while (poller.Next(&packet)) { - } - MP_ASSERT_OK(graph.WaitUntilDone()); - const VideoHeader& video_header = packet.Get(); - - // Checks the generated video file has the same width, height, fps, and - // duration as the original one. - cv::VideoCapture cap(output_file_path); - ASSERT_TRUE(cap.isOpened()); - EXPECT_EQ(video_header.width, - static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH))); - EXPECT_EQ(video_header.height, - static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - EXPECT_EQ(video_header.frame_rate, - static_cast(cap.get(cv::CAP_PROP_FPS))); - EXPECT_EQ(video_header.duration, - static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT) / - cap.get(cv::CAP_PROP_FPS))); -} - -TEST(OpenCvVideoEncoderCalculatorTest, TestFlvH264Video) { - CalculatorGraphConfig config = - ParseTextProtoOrDie(R"pb( - node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream" - } - node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:video" - input_stream: "VIDEO_PRESTREAM:video_prestream" - input_side_packet: "OUTPUT_FILE_PATH:output_file_path" - node_options { - [type.googleapis.com/ - mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "MJPG" - video_format: "avi" - } - } - } - )pb"); - std::map input_side_packets; - input_side_packets["input_file_path"] = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_FLV_H264_AAC.video")); - const std::string output_file_path = "/tmp/tmp_video.avi"; - DeletingFile deleting_file(output_file_path, true); - input_side_packets["output_file_path"] = - MakePacket(output_file_path); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config, input_side_packets)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("video_prestream"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - Packet packet; - while (poller.Next(&packet)) { - } - MP_ASSERT_OK(graph.WaitUntilDone()); - const VideoHeader& video_header = packet.Get(); - - // Checks the generated video file has the same width, height, fps, and - // duration as the original one. - cv::VideoCapture cap(output_file_path); - ASSERT_TRUE(cap.isOpened()); - EXPECT_EQ(video_header.width, - static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH))); - EXPECT_EQ(video_header.height, - static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - // TODO: The actual header.duration is 6.0666666f and the frame_rate - // can be either 30.30303f (with opencv2) or 30f (with opencv3 and opencv4). - // EXPECT_EQ(video_header.frame_rate, - // static_cast(cap.get(cv::CAP_PROP_FPS))); - // EXPECT_EQ(video_header.duration, - // static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT) / - // cap.get(cv::CAP_PROP_FPS))); -} - -TEST(OpenCvVideoEncoderCalculatorTest, TestMkvVp8Video) { - CalculatorGraphConfig config = - ParseTextProtoOrDie(R"pb( - node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_file_path" - output_stream: "VIDEO:video" - output_stream: "VIDEO_PRESTREAM:video_prestream" - } - node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:video" - input_stream: "VIDEO_PRESTREAM:video_prestream" - input_side_packet: "OUTPUT_FILE_PATH:output_file_path" - node_options { - [type.googleapis.com/ - mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "PIM1" - video_format: "mkv" - } - } - } - )pb"); - std::map input_side_packets; - input_side_packets["input_file_path"] = MakePacket( - file::JoinPath("./", - "/mediapipe/calculators/video/" - "testdata/format_MKV_VP8_VORBIS.video")); - const std::string output_file_path = "/tmp/tmp_video.mkv"; - DeletingFile deleting_file(output_file_path, true); - input_side_packets["output_file_path"] = - MakePacket(output_file_path); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config, input_side_packets)); - StatusOrPoller status_or_poller = - graph.AddOutputStreamPoller("video_prestream"); - ASSERT_TRUE(status_or_poller.ok()); - OutputStreamPoller poller = std::move(status_or_poller.value()); - - MP_ASSERT_OK(graph.StartRun({})); - Packet packet; - while (poller.Next(&packet)) { - } - MP_ASSERT_OK(graph.WaitUntilDone()); - const VideoHeader& video_header = packet.Get(); - - // Checks the generated video file has the same width, height, fps, and - // duration as the original one. - cv::VideoCapture cap(output_file_path); - ASSERT_TRUE(cap.isOpened()); - EXPECT_EQ(video_header.width, - static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH))); - EXPECT_EQ(video_header.height, - static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT))); - EXPECT_EQ(video_header.frame_rate, - static_cast(cap.get(cv::CAP_PROP_FPS))); - EXPECT_EQ(video_header.duration, - static_cast(std::round(cap.get(cv::CAP_PROP_FRAME_COUNT) / - cap.get(cv::CAP_PROP_FPS)))); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/video/testdata/format_FLV_H264_AAC.video b/mediapipe/calculators/video/testdata/format_FLV_H264_AAC.video deleted file mode 100644 index 980da555e..000000000 Binary files a/mediapipe/calculators/video/testdata/format_FLV_H264_AAC.video and /dev/null differ diff --git a/mediapipe/calculators/video/testdata/format_MKV_VP8_VORBIS.video b/mediapipe/calculators/video/testdata/format_MKV_VP8_VORBIS.video deleted file mode 100644 index ace0e8e6e..000000000 Binary files a/mediapipe/calculators/video/testdata/format_MKV_VP8_VORBIS.video and /dev/null differ diff --git a/mediapipe/calculators/video/testdata/format_MP4_AVC720P_AAC.video b/mediapipe/calculators/video/testdata/format_MP4_AVC720P_AAC.video deleted file mode 100644 index 4a9cafb94..000000000 Binary files a/mediapipe/calculators/video/testdata/format_MP4_AVC720P_AAC.video and /dev/null differ diff --git a/mediapipe/calculators/video/testdata/lenna.png b/mediapipe/calculators/video/testdata/lenna.png deleted file mode 100644 index bd247e8a4..000000000 Binary files a/mediapipe/calculators/video/testdata/lenna.png and /dev/null differ diff --git a/mediapipe/calculators/video/testdata/parallel_tracker_graph.pbtxt b/mediapipe/calculators/video/testdata/parallel_tracker_graph.pbtxt deleted file mode 100644 index f3fe5b424..000000000 --- a/mediapipe/calculators/video/testdata/parallel_tracker_graph.pbtxt +++ /dev/null @@ -1,134 +0,0 @@ -input_stream: "image_cpu_frames" -input_stream: "start_pos" -input_stream: "ra_track" - -num_threads: 4 - -node: { - calculator: "MotionAnalysisCalculator" - input_stream: "VIDEO:image_cpu_frames" - output_stream: "CAMERA:camera_motion" - output_stream: "FLOW:region_flow" - - options: { - [mediapipe.MotionAnalysisCalculatorOptions.ext]: { - analysis_options: { - analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE - - flow_options: { - # Maybe move down to 50 - fast_estimation_min_block_size: 100 - top_inlier_sets: 1 - frac_inlier_error_threshold: 3e-3 - downsample_mode: DOWNSAMPLE_NONE - verification_distance: 5.0 - verify_long_feature_acceleration: true - verify_long_feature_trigger_ratio: 0.1 - tracking_options: { - max_features: 500 - adaptive_extraction_levels: 2 - min_eig_val_settings: { - adaptive_lowest_quality_level: 2e-4 - } - klt_tracker_implementation: KLT_OPENCV - } - } - - motion_options: { - label_empty_frames_as_valid: false - } - } - } - } -} - -node: { - calculator: "FlowPackagerCalculator" - input_stream: "FLOW:region_flow" - input_stream: "CAMERA:camera_motion" - output_stream: "TRACKING:tracking_data" - - options: { - [mediapipe.FlowPackagerCalculatorOptions.ext]: { - flow_packager_options: { - binary_tracking_data_support: false - } - } - } -} - -node: { - calculator: "BoxTrackerCalculator" - input_stream: "TRACKING:tracking_data" - input_stream: "START_POS:start_pos" - output_stream: "BOXES:boxes" - input_side_packet: "OPTIONS:calculator_options" - - input_stream_handler: { - input_stream_handler: "SyncSetInputStreamHandler" - options: { - [mediapipe.SyncSetInputStreamHandlerOptions.ext]: { - sync_set: { - tag_index: "TRACKING" - } - sync_set: { - tag_index: "START_POS" - } - } - } - } - - options: { - [mediapipe.BoxTrackerCalculatorOptions.ext]: { - tracker_options: { - track_step_options: { - track_object_and_camera: true - tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE - object_similarity_min_contd_inliers: 6 - inlier_spring_force: 0.0 - static_motion_temporal_ratio: 3e-2 - } - } - visualize_tracking_data: false - streaming_track_data_cache_size: 100 - } - } -} - -node: { - calculator: "BoxTrackerCalculator" - input_stream: "TRACKING:tracking_data" - input_stream: "RA_TRACK:ra_track" - output_stream: "RA_BOXES:ra_boxes" - input_side_packet: "OPTIONS:calculator_options" - - input_stream_handler: { - input_stream_handler: "SyncSetInputStreamHandler" - options: { - [mediapipe.SyncSetInputStreamHandlerOptions.ext]: { - sync_set: { - tag_index: "TRACKING" - } - sync_set: { - tag_index: "RA_TRACK" - } - } - } - } - - options: { - [mediapipe.BoxTrackerCalculatorOptions.ext]: { - tracker_options: { - track_step_options: { - track_object_and_camera: true - tracking_degrees: TRACKING_DEGREE_OBJECT_PERSPECTIVE - object_similarity_min_contd_inliers: 6 - inlier_spring_force: 0.0 - static_motion_temporal_ratio: 3e-2 - } - } - visualize_tracking_data: false - streaming_track_data_cache_size: 100 - } - } -} diff --git a/mediapipe/calculators/video/testdata/tracker_graph.pbtxt b/mediapipe/calculators/video/testdata/tracker_graph.pbtxt deleted file mode 100644 index c86d50136..000000000 --- a/mediapipe/calculators/video/testdata/tracker_graph.pbtxt +++ /dev/null @@ -1,122 +0,0 @@ -input_stream: "image_cpu_frames" -input_stream: "start_pos" -input_stream: "cancel_object_id" -input_stream: "ra_track" -input_stream: "restart_pos" -input_stream: "track_time" - -num_threads: 4 - -node: { - calculator: "MotionAnalysisCalculator" - options: { - [mediapipe.MotionAnalysisCalculatorOptions.ext]: { - analysis_options: { - analysis_policy: ANALYSIS_POLICY_CAMERA_MOBILE - - flow_options: { - # Maybe move down to 50 - fast_estimation_min_block_size: 100 - top_inlier_sets: 1 - frac_inlier_error_threshold: 3e-3 - # For mobile application, downsample before input into graph - # and use DOWNSAMPLE_TO_INPUT_SIZE and specify - # downsampling_factor option or DOWNSAMPLE input_side_packet - downsample_mode: DOWNSAMPLE_TO_INPUT_SIZE - verification_distance: 5.0 - verify_long_feature_acceleration: true - verify_long_feature_trigger_ratio: 0.1 - tracking_options: { - max_features: 500 - corner_extraction_method: EXTRACTION_FAST - adaptive_extraction_levels: 2 - min_eig_val_settings: { - adaptive_lowest_quality_level: 2e-4 - } - klt_tracker_implementation: KLT_OPENCV - } - } - } - } - } - - # Drops packets if calculator cannot keep up with the input rate. - input_stream_handler: { - input_stream_handler: "FixedSizeInputStreamHandler" - } - - input_stream: "VIDEO:image_cpu_frames" - input_side_packet: "DOWNSAMPLE:analysis_downsample_factor" - input_side_packet: "OPTIONS:calculator_options" - output_stream: "CAMERA:camera_motion" - output_stream: "FLOW:region_flow" -} - -node: { - calculator: "FlowPackagerCalculator" - - input_stream: "FLOW:region_flow" - input_stream: "CAMERA:camera_motion" - output_stream: "TRACKING:tracking_data" - options: { - [mediapipe.FlowPackagerCalculatorOptions.ext]: { - flow_packager_options: { - binary_tracking_data_support: false - } - } - } -} - -node: { - calculator: "BoxTrackerCalculator" - - input_side_packet: "OPTIONS:calculator_options" - input_stream: "TRACKING:tracking_data" - input_stream: "TRACK_TIME:track_time" - input_stream: "START_POS:start_pos" - input_stream: "RESTART_POS:restart_pos" - input_stream: "CANCEL_OBJECT_ID:cancel_object_id" - input_stream: "RA_TRACK:ra_track" - output_stream: "BOXES:boxes" - output_stream: "RA_BOXES:ra_boxes" - - input_stream_handler: { - input_stream_handler: "SyncSetInputStreamHandler" - options: { - [mediapipe.SyncSetInputStreamHandlerOptions.ext]: { - sync_set: { - tag_index: "TRACKING" - tag_index: "TRACK_TIME" - } - sync_set: { - tag_index: "START_POS" - } - sync_set: { - tag_index: "RESTART_POS" - } - sync_set: { - tag_index: "CANCEL_OBJECT_ID" - } - sync_set: { - tag_index: "RA_TRACK" - } - } - } - } - - options: { - [mediapipe.BoxTrackerCalculatorOptions.ext]: { - tracker_options: { - track_step_options: { - track_object_and_camera: true - tracking_degrees: TRACKING_DEGREE_OBJECT_ROTATION_SCALE - inlier_spring_force: 0.0 - static_motion_temporal_ratio: 3e-2 - object_similarity_min_contd_inliers: 10 - } - } - visualize_tracking_data: false - streaming_track_data_cache_size: 100 - } - } -} diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD deleted file mode 100644 index 408461d2f..000000000 --- a/mediapipe/calculators/video/tool/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"]) - -proto_library( - name = "flow_quantizer_model_proto", - srcs = ["flow_quantizer_model.proto"], - visibility = ["//visibility:public"], -) - -mediapipe_cc_proto_library( - name = "flow_quantizer_model_cc_proto", - srcs = ["flow_quantizer_model.proto"], - visibility = ["//visibility:public"], - deps = [":flow_quantizer_model_proto"], -) - -cc_library( - name = "flow_quantizer_model", - srcs = ["flow_quantizer_model.cc"], - hdrs = ["flow_quantizer_model.h"], - deps = [ - "//mediapipe/calculators/video/tool:flow_quantizer_model_cc_proto", - "//mediapipe/framework:type_map", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats/motion:optical_flow_field", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/tool:status_util", - ], - alwayslink = 1, -) diff --git a/mediapipe/calculators/video/tool/flow_quantizer_model.cc b/mediapipe/calculators/video/tool/flow_quantizer_model.cc deleted file mode 100644 index 0cfad8539..000000000 --- a/mediapipe/calculators/video/tool/flow_quantizer_model.cc +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/video/tool/flow_quantizer_model.h" - -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/type_map.h" - -namespace mediapipe { - -// Uniform normalization to 0-255. -uint8 FlowQuantizerModel::Apply(const float val, const int channel) const { - CHECK_LT(channel, model_.min_value_size()); - const auto& min_value = model_.min_value(channel); - const auto& max_value = model_.max_value(channel); - QCHECK_GT(max_value, min_value); - float res = (val - min_value) / (max_value - min_value); - if (res < 0.0) { - res = 0.0; - } else if (res > 1.0) { - res = 1.0; - } - return static_cast(res * 255); -} - -void FlowQuantizerModel::LoadFromProto(const QuantizerModelData& data) { - QCHECK_GT(data.max_value(0), data.min_value(0)); - QCHECK_GT(data.max_value(1), data.min_value(1)); - - model_ = data; -} - -const QuantizerModelData& FlowQuantizerModel::GetModelData() const { - return model_; -} - -// Used for training, update the (min, max) range. We want to estimate the range -// of optical flow fields (Theorectically it is (-num_pixels_along_diag, -// num_pixels_along_diag). -// TODO: Taking the min and max over all training flow fields might be -// sensitive to noise. We should use more robust statistics. -void FlowQuantizerModel::AddSampleFlowField(const OpticalFlowField& flow) { - CHECK_EQ(model_.min_value_size(), 2); - const cv::Mat_& flow_mat = flow.flow_data(); - for (int i = 0; i != flow.width(); ++i) { - for (int j = 0; j != flow.height(); ++j) { - const auto& x = flow_mat.at(i, j).x; - const auto& y = flow_mat.at(i, j).y; - // Always use the minimum and maximum value occurred in training flow - // fields. - model_.set_min_value(0, std::min(x, model_.min_value(0))); - model_.set_min_value(1, std::min(y, model_.min_value(1))); - model_.set_max_value(0, std::max(x, model_.max_value(0))); - model_.set_max_value(1, std::max(y, model_.max_value(1))); - } - } -} - -void FlowQuantizerModel::Init() { - model_.Clear(); - // Initialize the values. - for (int i = 0; i != 2; ++i) { - model_.add_min_value(std::numeric_limits::max()); - model_.add_max_value(-std::numeric_limits::max()); - } -} -} // namespace mediapipe diff --git a/mediapipe/calculators/video/tool/flow_quantizer_model.h b/mediapipe/calculators/video/tool/flow_quantizer_model.h deleted file mode 100644 index 16ae4b7ac..000000000 --- a/mediapipe/calculators/video/tool/flow_quantizer_model.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Quantization model to convert a real value float number (flow field) to a -// 8-bit discrete number. -#ifndef MEDIAPIPE_CALCULATORS_VIDEO_TOOL_FLOW_QUANTIZER_MODEL_H_ -#define MEDIAPIPE_CALCULATORS_VIDEO_TOOL_FLOW_QUANTIZER_MODEL_H_ - -#include "mediapipe/calculators/video/tool/flow_quantizer_model.pb.h" -#include "mediapipe/framework/formats/motion/optical_flow_field.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/tool/status_util.h" - -namespace mediapipe { - -class FlowQuantizerModel { - public: - // Initializes the model proto. - void Init(); - // Quantizes flow field with the model. - uint8 Apply(const float val, const int channel) const; - // Loads model from proto. - void LoadFromProto(const QuantizerModelData& data); - // Gets proto from model. - const QuantizerModelData& GetModelData() const; - // Used in training. Updates the model proto by reading the flow fields. - // TODO: This model is currently manually set. Need to find a way to - // learn from flow fields directly. - void AddSampleFlowField(const OpticalFlowField& flow); - - private: - QuantizerModelData model_; -}; -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_VIDEO_TOOL_FLOW_QUANTIZER_MODEL_H_ diff --git a/mediapipe/calculators/video/tool/flow_quantizer_model.proto b/mediapipe/calculators/video/tool/flow_quantizer_model.proto deleted file mode 100644 index 02f3fc11a..000000000 --- a/mediapipe/calculators/video/tool/flow_quantizer_model.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -// Message storing min value and max value for normalization in all channels. -message QuantizerModelData { - // For all channels. - repeated float min_value = 1; - repeated float max_value = 2; -} diff --git a/mediapipe/calculators/video/tracked_detection_manager_calculator.cc b/mediapipe/calculators/video/tracked_detection_manager_calculator.cc deleted file mode 100644 index c416fa9b0..000000000 --- a/mediapipe/calculators/video/tracked_detection_manager_calculator.cc +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#include "absl/container/node_hash_map.h" -#include "mediapipe/calculators/video/tracked_detection_manager_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/util/tracking/box_tracker.h" -#include "mediapipe/util/tracking/tracked_detection.h" -#include "mediapipe/util/tracking/tracked_detection_manager.h" -#include "mediapipe/util/tracking/tracking.h" - -namespace mediapipe { -namespace { - -constexpr int kDetectionUpdateTimeOutMS = 5000; -constexpr char kDetectionsTag[] = "DETECTIONS"; -constexpr char kDetectionBoxesTag[] = "DETECTION_BOXES"; -constexpr char kDetectionListTag[] = "DETECTION_LIST"; -constexpr char kTrackingBoxesTag[] = "TRACKING_BOXES"; -constexpr char kCancelObjectIdTag[] = "CANCEL_OBJECT_ID"; - -// Move |src| to the back of |dst|. -void MoveIds(std::vector* dst, std::vector src) { - dst->insert(dst->end(), std::make_move_iterator(src.begin()), - std::make_move_iterator(src.end())); -} - -int64 GetInputTimestampMs(::mediapipe::CalculatorContext* cc) { - return cc->InputTimestamp().Microseconds() / 1000; // 1 ms = 1000 us. -} - -// Converts a Mediapipe Detection Proto to a TrackedDetection class. -std::unique_ptr GetTrackedDetectionFromDetection( - const Detection& detection, int64 timestamp) { - std::unique_ptr tracked_detection = - absl::make_unique(detection.detection_id(), timestamp); - const float top = detection.location_data().relative_bounding_box().ymin(); - const float bottom = - detection.location_data().relative_bounding_box().ymin() + - detection.location_data().relative_bounding_box().height(); - const float left = detection.location_data().relative_bounding_box().xmin(); - const float right = detection.location_data().relative_bounding_box().xmin() + - detection.location_data().relative_bounding_box().width(); - NormalizedRect bounding_box; - bounding_box.set_x_center((left + right) / 2.f); - bounding_box.set_y_center((top + bottom) / 2.f); - bounding_box.set_height(bottom - top); - bounding_box.set_width(right - left); - tracked_detection->set_bounding_box(bounding_box); - - for (int i = 0; i < detection.label_size(); ++i) { - tracked_detection->AddLabel(detection.label(i), detection.score(i)); - } - return tracked_detection; -} - -// Converts a TrackedDetection class to a Mediapipe Detection Proto. -Detection GetAxisAlignedDetectionFromTrackedDetection( - const TrackedDetection& tracked_detection) { - Detection detection; - LocationData* location_data = detection.mutable_location_data(); - - auto corners = tracked_detection.GetCorners(); - - float x_min = std::numeric_limits::max(); - float x_max = std::numeric_limits::min(); - float y_min = std::numeric_limits::max(); - float y_max = std::numeric_limits::min(); - for (int i = 0; i < 4; ++i) { - x_min = std::min(x_min, corners[i].x()); - x_max = std::max(x_max, corners[i].x()); - y_min = std::min(y_min, corners[i].y()); - y_max = std::max(y_max, corners[i].y()); - } - location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); - LocationData::RelativeBoundingBox* relative_bbox = - location_data->mutable_relative_bounding_box(); - relative_bbox->set_xmin(x_min); - relative_bbox->set_ymin(y_min); - relative_bbox->set_width(x_max - x_min); - relative_bbox->set_height(y_max - y_min); - - // Use previous id which is the id the object when it's first detected. - if (tracked_detection.previous_id() > 0) { - detection.set_detection_id(tracked_detection.previous_id()); - } else { - detection.set_detection_id(tracked_detection.unique_id()); - } - - // Sort the labels by descending scores. - std::vector> labels_and_scores; - for (const auto& label_and_score : tracked_detection.label_to_score_map()) { - labels_and_scores.push_back(label_and_score); - } - std::sort(labels_and_scores.begin(), labels_and_scores.end(), - [](const auto& a, const auto& b) { return a.second > b.second; }); - for (const auto& label_and_score : labels_and_scores) { - detection.add_label(label_and_score.first); - detection.add_score(label_and_score.second); - } - return detection; -} - -} // namespace - -// TrackedDetectionManagerCalculator accepts detections and tracking results at -// different frame rate for real time tracking of targets. -// Input: -// DETECTIONS: A vector of newly detected targets. -// TRACKING_BOXES: A TimedBoxProtoList which contains a list of tracked boxes -// from previous detections. -// -// Output: -// CANCEL_OBJECT_ID: Ids of targets that are missing/lost such that it should -// be removed from tracking. -// DETECTIONS: List of detections that are being tracked. -// DETECTION_BOXES: List of bounding boxes of detections that are being -// tracked. -// -// Usage example: -// node { -// calculator: "TrackedDetectionManagerCalculator" -// input_stream: "DETECTIONS:detections" -// input_stream: "TRACKING_BOXES:boxes" -// output_stream: "CANCEL_OBJECT_ID:cancel_object_id" -// output_stream: "DETECTIONS:output_detections" -// } -class TrackedDetectionManagerCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - - absl::Status Process(CalculatorContext* cc) override; - - private: - // Adds new list of detections to |waiting_for_update_detections_|. - void AddDetectionList(const DetectionList& detection_list, - CalculatorContext* cc); - void AddDetections(const std::vector& detections, - CalculatorContext* cc); - - // Manages existing and new detections. - TrackedDetectionManager tracked_detection_manager_; - - // Set of detections that are not up to date yet. These detections will be - // added to the detection manager until they got updated from the box tracker. - absl::node_hash_map> - waiting_for_update_detections_; -}; -REGISTER_CALCULATOR(TrackedDetectionManagerCalculator); - -absl::Status TrackedDetectionManagerCalculator::GetContract( - CalculatorContract* cc) { - if (cc->Inputs().HasTag(kDetectionsTag)) { - cc->Inputs().Tag(kDetectionsTag).Set>(); - } - if (cc->Inputs().HasTag(kDetectionListTag)) { - cc->Inputs().Tag(kDetectionListTag).Set(); - } - if (cc->Inputs().HasTag(kTrackingBoxesTag)) { - cc->Inputs().Tag(kTrackingBoxesTag).Set(); - } - - if (cc->Outputs().HasTag(kCancelObjectIdTag)) { - cc->Outputs().Tag(kCancelObjectIdTag).Set(); - } - if (cc->Outputs().HasTag(kDetectionsTag)) { - cc->Outputs().Tag(kDetectionsTag).Set>(); - } - if (cc->Outputs().HasTag(kDetectionBoxesTag)) { - cc->Outputs().Tag(kDetectionBoxesTag).Set>(); - } - - return absl::OkStatus(); -} - -absl::Status TrackedDetectionManagerCalculator::Open(CalculatorContext* cc) { - mediapipe::TrackedDetectionManagerCalculatorOptions options = - cc->Options(); - tracked_detection_manager_.SetConfig( - options.tracked_detection_manager_options()); - return absl::OkStatus(); -} - -absl::Status TrackedDetectionManagerCalculator::Process(CalculatorContext* cc) { - if (cc->Inputs().HasTag(kTrackingBoxesTag) && - !cc->Inputs().Tag(kTrackingBoxesTag).IsEmpty()) { - const TimedBoxProtoList& tracked_boxes = - cc->Inputs().Tag(kTrackingBoxesTag).Get(); - - // Collect all detections that are removed. - auto removed_detection_ids = absl::make_unique>(); - for (const TimedBoxProto& tracked_box : tracked_boxes.box()) { - NormalizedRect bounding_box; - bounding_box.set_x_center((tracked_box.left() + tracked_box.right()) / - 2.f); - bounding_box.set_y_center((tracked_box.bottom() + tracked_box.top()) / - 2.f); - bounding_box.set_height(tracked_box.bottom() - tracked_box.top()); - bounding_box.set_width(tracked_box.right() - tracked_box.left()); - bounding_box.set_rotation(tracked_box.rotation()); - // First check if this box updates a detection that's waiting for - // update from the tracker. - auto waiting_for_update_detectoin_ptr = - waiting_for_update_detections_.find(tracked_box.id()); - if (waiting_for_update_detectoin_ptr != - waiting_for_update_detections_.end()) { - // Add the detection and remove duplicated detections. - auto removed_ids = tracked_detection_manager_.AddDetection( - std::move(waiting_for_update_detectoin_ptr->second)); - MoveIds(removed_detection_ids.get(), std::move(removed_ids)); - - waiting_for_update_detections_.erase(waiting_for_update_detectoin_ptr); - } - auto removed_ids = tracked_detection_manager_.UpdateDetectionLocation( - tracked_box.id(), bounding_box, tracked_box.time_msec()); - MoveIds(removed_detection_ids.get(), std::move(removed_ids)); - } - // TODO: Should be handled automatically in detection manager. - auto removed_ids = tracked_detection_manager_.RemoveObsoleteDetections( - GetInputTimestampMs(cc) - kDetectionUpdateTimeOutMS); - MoveIds(removed_detection_ids.get(), std::move(removed_ids)); - - // TODO: Should be handled automatically in detection manager. - removed_ids = tracked_detection_manager_.RemoveOutOfViewDetections(); - MoveIds(removed_detection_ids.get(), std::move(removed_ids)); - - if (!removed_detection_ids->empty() && - cc->Outputs().HasTag(kCancelObjectIdTag)) { - auto timestamp = cc->InputTimestamp(); - for (int box_id : *removed_detection_ids) { - // The timestamp is incremented (by 1 us) because currently the box - // tracker calculator only accepts one cancel object ID for any given - // timestamp. - cc->Outputs() - .Tag(kCancelObjectIdTag) - .AddPacket(mediapipe::MakePacket(box_id).At(timestamp++)); - } - } - - // Output detections and corresponding bounding boxes. - const auto& all_detections = - tracked_detection_manager_.GetAllTrackedDetections(); - auto output_detections = absl::make_unique>(); - auto output_boxes = absl::make_unique>(); - - for (const auto& detection_ptr : all_detections) { - const auto& detection = *detection_ptr.second; - // Only output detections that are synced. - if (detection.last_updated_timestamp() < - cc->InputTimestamp().Microseconds() / 1000) { - continue; - } - output_detections->emplace_back( - GetAxisAlignedDetectionFromTrackedDetection(detection)); - output_boxes->emplace_back(detection.bounding_box()); - } - if (cc->Outputs().HasTag(kDetectionsTag)) { - cc->Outputs() - .Tag(kDetectionsTag) - .Add(output_detections.release(), cc->InputTimestamp()); - } - - if (cc->Outputs().HasTag(kDetectionBoxesTag)) { - cc->Outputs() - .Tag(kDetectionBoxesTag) - .Add(output_boxes.release(), cc->InputTimestamp()); - } - } - - if (cc->Inputs().HasTag(kDetectionsTag) && - !cc->Inputs().Tag(kDetectionsTag).IsEmpty()) { - const auto detections = - cc->Inputs().Tag(kDetectionsTag).Get>(); - AddDetections(detections, cc); - } - - if (cc->Inputs().HasTag(kDetectionListTag) && - !cc->Inputs().Tag(kDetectionListTag).IsEmpty()) { - const auto detection_list = - cc->Inputs().Tag(kDetectionListTag).Get(); - AddDetectionList(detection_list, cc); - } - - return absl::OkStatus(); -} - -void TrackedDetectionManagerCalculator::AddDetectionList( - const DetectionList& detection_list, CalculatorContext* cc) { - for (const auto& detection : detection_list.detection()) { - // Convert from microseconds to milliseconds. - std::unique_ptr new_detection = - GetTrackedDetectionFromDetection( - detection, cc->InputTimestamp().Microseconds() / 1000); - - const int id = new_detection->unique_id(); - waiting_for_update_detections_[id] = std::move(new_detection); - } -} - -void TrackedDetectionManagerCalculator::AddDetections( - const std::vector& detections, CalculatorContext* cc) { - for (const auto& detection : detections) { - // Convert from microseconds to milliseconds. - std::unique_ptr new_detection = - GetTrackedDetectionFromDetection( - detection, cc->InputTimestamp().Microseconds() / 1000); - - const int id = new_detection->unique_id(); - waiting_for_update_detections_[id] = std::move(new_detection); - } -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/tracked_detection_manager_calculator.proto b/mediapipe/calculators/video/tracked_detection_manager_calculator.proto deleted file mode 100644 index 446242588..000000000 --- a/mediapipe/calculators/video/tracked_detection_manager_calculator.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; -import "mediapipe/util/tracking/tracked_detection_manager_config.proto"; - -message TrackedDetectionManagerCalculatorOptions { - extend CalculatorOptions { - optional TrackedDetectionManagerCalculatorOptions ext = 301970230; - } - - optional TrackedDetectionManagerConfig tracked_detection_manager_options = 1; -} diff --git a/mediapipe/calculators/video/tracking_graph_test.cc b/mediapipe/calculators/video/tracking_graph_test.cc deleted file mode 100644 index e446e155c..000000000 --- a/mediapipe/calculators/video/tracking_graph_test.cc +++ /dev/null @@ -1,709 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -#include "mediapipe/calculators/video/box_tracker_calculator.pb.h" -#include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/packet.h" -#include "mediapipe/framework/port/advanced_proto_inc.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/proto_ns.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/tracking/box_tracker.pb.h" -#include "mediapipe/util/tracking/tracking.pb.h" - -#ifdef __APPLE__ -#include -#endif // defined(__APPLE__) - -namespace mediapipe { -namespace { -using ::testing::FloatNear; -using ::testing::Test; - -std::string GetTestDir() { -#ifdef __APPLE__ - char path[1024]; - CFURLRef bundle_url = CFBundleCopyBundleURL(CFBundleGetMainBundle()); - CFURLGetFileSystemRepresentation( - bundle_url, true, reinterpret_cast(path), sizeof(path)); - CFRelease(bundle_url); - return mediapipe::file::JoinPath(path, "testdata"); -#elif defined(__ANDROID__) - char path[1024]; - getcwd(path, sizeof(path)); - return mediapipe::file::JoinPath(path, - "mediapipe/calculators/video/testdata"); -#else - return mediapipe::file::JoinPath( - "./", - // This should match the path of the output files - // of the genrule() that generates test model files. - "mediapipe/calculators/video/testdata"); -#endif // defined(__APPLE__) -} - -bool LoadBinaryTestGraph(const std::string& graph_path, - CalculatorGraphConfig* config) { - std::ifstream ifs; - ifs.open(graph_path.c_str()); - proto_ns::io::IstreamInputStream in_stream(&ifs); - bool success = config->ParseFromZeroCopyStream(&in_stream); - ifs.close(); - if (!success) { - LOG(ERROR) << "could not parse test graph: " << graph_path; - } - return success; -} - -class TrackingGraphTest : public Test { - protected: - TrackingGraphTest() {} - - void SetUp() override { - test_dir_ = GetTestDir(); - const auto graph_path = file::JoinPath(test_dir_, "tracker.binarypb"); - ASSERT_TRUE(LoadBinaryTestGraph(graph_path, &config_)); - - original_image_ = cv::imread(file::JoinPath(test_dir_, "lenna.png")); - CreateInputFramesFromOriginalImage(kNumImages, kTranslationStep, - &input_frames_packets_); - - const auto& first_input_img = input_frames_packets_[0].Get(); - const int img_width = first_input_img.Width(); - const int img_height = first_input_img.Height(); - translation_step_x_ = kTranslationStep / static_cast(img_width); - translation_step_y_ = kTranslationStep / static_cast(img_height); - - // Creat new configure and packet dump vector to store output. - mediapipe::CalculatorGraphConfig config_copy = config_; - mediapipe::tool::AddVectorSink("boxes", &config_copy, &output_packets_); - mediapipe::tool::AddVectorSink("ra_boxes", &config_copy, - &random_access_results_packets_); - - // Initialize graph - MP_ASSERT_OK(graph_.Initialize(config_copy)); - - const auto parallel_graph_path = - file::JoinPath(test_dir_, "parallel_tracker.binarypb"); - CalculatorGraphConfig parallel_config; - ASSERT_TRUE(LoadBinaryTestGraph(parallel_graph_path, ¶llel_config)); - mediapipe::tool::AddVectorSink("boxes", ¶llel_config, &output_packets_); - mediapipe::tool::AddVectorSink("ra_boxes", ¶llel_config, - &random_access_results_packets_); - MP_ASSERT_OK(parallel_graph_.Initialize(parallel_config)); - } - - void CreateInputFramesFromOriginalImage( - int num_images, int translation_step, - std::vector* input_frames_packets); - - void TearDown() override { - output_packets_.clear(); - random_access_results_packets_.clear(); - } - - std::unique_ptr MakeBoxList( - const Timestamp& timestamp, const std::vector& is_quad_tracking, - const std::vector& is_pnp_tracking, - const std::vector& reacquisition) const; - - void RunGraphWithSidePacketsAndInputs( - const std::map& side_packets, - const mediapipe::Packet& start_pos_packet); - - // Utility functions used to judge if a given quad or box is near to the - // groundtruth location at a given frame. - // Examine box.reacquisition() field equals to `reacquisition`. - // `frame` can be float number to account for inter-frame interpolation. - void ExpectBoxAtFrame(const TimedBoxProto& box, float frame, - bool reacquisition); - - // Examine box.aspect_ratio() field equals to `aspect_ratio` if asepct_ratio - // is positive. - void ExpectQuadAtFrame(const TimedBoxProto& box, float frame, - float aspect_ratio, bool reacquisition); - - // Utility function to judge if two quad are near to each other. - void ExpectQuadNear(const TimedBoxProto& box1, const TimedBoxProto& box2); - - std::unique_ptr CreateRandomAccessTrackingBoxList( - const std::vector& start_timestamps, - const std::vector& end_timestamps) const; - - CalculatorGraph graph_; - CalculatorGraph parallel_graph_; - CalculatorGraphConfig config_; - std::string test_dir_; - cv::Mat original_image_; - std::vector input_frames_packets_; - std::vector output_packets_; - std::vector random_access_results_packets_; - float translation_step_x_; // normalized translation step in x direction - float translation_step_y_; // normalized translation step in y direction - static constexpr float kInitialBoxHalfWidthNormalized = 0.25f; - static constexpr float kInitialBoxHalfHeightNormalized = 0.25f; - static constexpr float kImageAspectRatio = 1.0f; // for lenna.png - static constexpr float kInitialBoxLeft = - 0.5f - kInitialBoxHalfWidthNormalized; - static constexpr float kInitialBoxRight = - 0.5f + kInitialBoxHalfWidthNormalized; - static constexpr float kInitialBoxTop = - 0.5f - kInitialBoxHalfHeightNormalized; - static constexpr float kInitialBoxBottom = - 0.5f + kInitialBoxHalfHeightNormalized; - static constexpr int kFrameIntervalUs = 30000; - static constexpr int kNumImages = 8; - // Each image is shifted to the right and bottom by kTranslationStep - // pixels compared with the previous image. - static constexpr int kTranslationStep = 10; - static constexpr float kEqualityTolerance = 3e-4f; -}; - -void TrackingGraphTest::ExpectBoxAtFrame(const TimedBoxProto& box, float frame, - bool reacquisition) { - EXPECT_EQ(box.reacquisition(), reacquisition); - EXPECT_TRUE(box.has_rotation()); - EXPECT_THAT(box.rotation(), FloatNear(0, kEqualityTolerance)); - EXPECT_THAT(box.left(), - FloatNear(kInitialBoxLeft - frame * translation_step_x_, - kEqualityTolerance)); - EXPECT_THAT(box.top(), FloatNear(kInitialBoxTop - frame * translation_step_y_, - kEqualityTolerance)); - EXPECT_THAT(box.bottom(), - FloatNear(kInitialBoxBottom - frame * translation_step_y_, - kEqualityTolerance)); - EXPECT_THAT(box.right(), - FloatNear(kInitialBoxRight - frame * translation_step_x_, - kEqualityTolerance)); -} - -void TrackingGraphTest::ExpectQuadAtFrame(const TimedBoxProto& box, float frame, - float aspect_ratio, - bool reacquisition) { - EXPECT_TRUE(box.has_quad()) << "quad must exist!"; - if (aspect_ratio > 0) { - EXPECT_TRUE(box.has_aspect_ratio()); - EXPECT_NEAR(box.aspect_ratio(), aspect_ratio, kEqualityTolerance); - } - - EXPECT_EQ(box.reacquisition(), reacquisition); - - const auto& quad = box.quad(); - EXPECT_EQ(8, quad.vertices_size()) - << "quad has only " << box.quad().vertices_size() << " vertices"; - EXPECT_THAT(quad.vertices(0), - FloatNear(kInitialBoxLeft - frame * translation_step_x_, - kEqualityTolerance)); - EXPECT_THAT(quad.vertices(1), - FloatNear(kInitialBoxTop - frame * translation_step_y_, - kEqualityTolerance)); - EXPECT_THAT(quad.vertices(3), - FloatNear(kInitialBoxBottom - frame * translation_step_y_, - kEqualityTolerance)); - EXPECT_THAT(quad.vertices(4), - FloatNear(kInitialBoxRight - frame * translation_step_x_, - kEqualityTolerance)); -} - -void TrackingGraphTest::ExpectQuadNear(const TimedBoxProto& box1, - const TimedBoxProto& box2) { - EXPECT_TRUE(box1.has_quad()); - EXPECT_TRUE(box2.has_quad()); - EXPECT_EQ(8, box1.quad().vertices_size()) - << "quad has only " << box1.quad().vertices_size() << " vertices"; - EXPECT_EQ(8, box2.quad().vertices_size()) - << "quad has only " << box2.quad().vertices_size() << " vertices"; - for (int j = 0; j < box1.quad().vertices_size(); ++j) { - EXPECT_NEAR(box1.quad().vertices(j), box2.quad().vertices(j), - kEqualityTolerance); - } -} - -std::unique_ptr TrackingGraphTest::MakeBoxList( - const Timestamp& timestamp, const std::vector& is_quad_tracking, - const std::vector& is_pnp_tracking, - const std::vector& reacquisition) const { - auto box_list = absl::make_unique(); - int box_id = 0; - for (int j = 0; j < is_quad_tracking.size(); ++j) { - TimedBoxProto* box = box_list->add_box(); - if (is_quad_tracking[j]) { - box->mutable_quad()->add_vertices(kInitialBoxLeft); - box->mutable_quad()->add_vertices(kInitialBoxTop); - box->mutable_quad()->add_vertices(kInitialBoxLeft); - box->mutable_quad()->add_vertices(kInitialBoxBottom); - box->mutable_quad()->add_vertices(kInitialBoxRight); - box->mutable_quad()->add_vertices(kInitialBoxBottom); - box->mutable_quad()->add_vertices(kInitialBoxRight); - box->mutable_quad()->add_vertices(kInitialBoxTop); - - if (is_pnp_tracking[j]) { - box->set_aspect_ratio(kImageAspectRatio); - } - } else { - box->set_left(kInitialBoxLeft); - box->set_right(kInitialBoxRight); - box->set_top(kInitialBoxTop); - box->set_bottom(kInitialBoxBottom); - } - box->set_id(box_id++); - box->set_time_msec(timestamp.Value() / 1000); - box->set_reacquisition(reacquisition[j]); - } - - return box_list; -} - -void TrackingGraphTest::CreateInputFramesFromOriginalImage( - int num_images, int translation_step, - std::vector* input_frames_packets) { - const int crop_width = original_image_.cols - num_images * translation_step; - const int crop_height = original_image_.rows - num_images * translation_step; - for (int i = 0; i < num_images; ++i) { - cv::Rect roi(i * translation_step, i * translation_step, crop_width, - crop_height); - cv::Mat cropped_img = cv::Mat(original_image_, roi); - auto cropped_image_frame = absl::make_unique( - ImageFormat::SRGB, crop_width, crop_height, cropped_img.step[0], - cropped_img.data, ImageFrame::PixelDataDeleter::kNone); - Timestamp curr_timestamp = Timestamp(i * kFrameIntervalUs); - Packet image_packet = - Adopt(cropped_image_frame.release()).At(curr_timestamp); - input_frames_packets->push_back(image_packet); - } -} - -void TrackingGraphTest::RunGraphWithSidePacketsAndInputs( - const std::map& side_packets, - const mediapipe::Packet& start_pos_packet) { - // Start running the graph - MP_EXPECT_OK(graph_.StartRun(side_packets)); - - MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet)); - - for (auto frame_packet : input_frames_packets_) { - MP_EXPECT_OK( - graph_.AddPacketToInputStream("image_cpu_frames", frame_packet)); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - } - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); -} - -std::unique_ptr -TrackingGraphTest::CreateRandomAccessTrackingBoxList( - const std::vector& start_timestamps, - const std::vector& end_timestamps) const { - CHECK_EQ(start_timestamps.size(), end_timestamps.size()); - auto ra_boxes = absl::make_unique(); - for (int i = 0; i < start_timestamps.size(); ++i) { - auto start_box_list = - MakeBoxList(start_timestamps[i], std::vector{true}, - std::vector{true}, std::vector{false}); - auto end_box_list = - MakeBoxList(end_timestamps[i], std::vector{true}, - std::vector{true}, std::vector{false}); - *(ra_boxes->add_box()) = (*start_box_list).box(0); - *(ra_boxes->add_box()) = (*end_box_list).box(0); - } - return ra_boxes; -} - -TEST_F(TrackingGraphTest, BasicBoxTrackingSanityCheck) { - // Create input side packets. - std::map side_packets; - side_packets.insert(std::make_pair("analysis_downsample_factor", - mediapipe::MakePacket(1.0f))); - side_packets.insert(std::make_pair( - "calculator_options", - mediapipe::MakePacket(CalculatorOptions()))); - - // Run the graph with input side packets, start_pos, and input image frames. - Timestamp start_box_time = input_frames_packets_[0].Timestamp(); - // is_quad_tracking is used to indicate whether to track quad for each - // individual box. - std::vector is_quad_tracking{false}; - // is_pnp_tracking is used to indicate whether to use perspective transform to - // track quad. - std::vector is_pnp_tracking{false}; - // is_reacquisition is used to indicate whether to enable reacquisition for - // the box. - std::vector is_reacquisition{false}; - auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking, - is_pnp_tracking, is_reacquisition); - Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time); - RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet); - - EXPECT_EQ(input_frames_packets_.size(), output_packets_.size()); - - for (int i = 0; i < output_packets_.size(); ++i) { - const TimedBoxProtoList& boxes = - output_packets_[i].Get(); - EXPECT_EQ(is_quad_tracking.size(), boxes.box_size()); - ExpectBoxAtFrame(boxes.box(0), i, false); - } -} - -TEST_F(TrackingGraphTest, BasicQuadTrackingSanityCheck) { - // Create input side packets. - std::map side_packets; - side_packets.insert(std::make_pair("analysis_downsample_factor", - mediapipe::MakePacket(1.0f))); - CalculatorOptions calculator_options; - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->mutable_tracker_options() - ->mutable_track_step_options() - ->set_tracking_degrees( - TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE); - side_packets.insert(std::make_pair( - "calculator_options", - mediapipe::MakePacket(calculator_options))); - - Timestamp start_box_time = input_frames_packets_[0].Timestamp(); - // Box id 0 use quad tracking with 8DoF homography transform. - // Box id 1 use quad tracking with 6DoF perspective transform. - // Box id 2 use box tracking with 4DoF similarity transform. - std::vector is_quad_tracking{true, true, false}; - std::vector is_pnp_tracking{false, true, false}; - std::vector is_reacquisition{true, false, true}; - auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking, - is_pnp_tracking, is_reacquisition); - Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time); - RunGraphWithSidePacketsAndInputs(side_packets, start_pos_packet); - - EXPECT_EQ(input_frames_packets_.size(), output_packets_.size()); - for (int i = 0; i < output_packets_.size(); ++i) { - const TimedBoxProtoList& boxes = - output_packets_[i].Get(); - EXPECT_EQ(is_quad_tracking.size(), boxes.box_size()); - for (int j = 0; j < boxes.box_size(); ++j) { - const TimedBoxProto& box = boxes.box(j); - if (is_quad_tracking[box.id()]) { - ExpectQuadAtFrame(box, i, - is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f, - is_reacquisition[box.id()]); - } else { - ExpectBoxAtFrame(box, i, is_reacquisition[box.id()]); - } - } - } -} - -TEST_F(TrackingGraphTest, TestRandomAccessTrackingResults) { - // Create input side packets. - std::map side_packets; - side_packets.insert(std::make_pair("analysis_downsample_factor", - mediapipe::MakePacket(1.0f))); - CalculatorOptions calculator_options; - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->mutable_tracker_options() - ->mutable_track_step_options() - ->set_tracking_degrees( - TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE); - side_packets.insert(std::make_pair( - "calculator_options", - mediapipe::MakePacket(calculator_options))); - - ASSERT_GT(input_frames_packets_.size(), 2); // at least 3 frames - ASSERT_TRUE(input_frames_packets_[2].Timestamp() - - input_frames_packets_[1].Timestamp() > - TimestampDiff(1000)); - - constexpr int start_frame = 0; - Timestamp start_box_time = input_frames_packets_[start_frame].Timestamp(); - auto start_box_list = - MakeBoxList(start_box_time, std::vector{true}, - std::vector{true}, std::vector{false}); - constexpr int end_frame = 2; - Timestamp end_box_time = input_frames_packets_[end_frame].Timestamp(); - - // Also test reverse random access tracking. - // This offset of 1ms is simulating the case where the start query timestamp - // to be not any existing frame timestamp. In reality, it's highly encouraged - // to have the start query timestamp be aligned with frame timestamp. - constexpr int reverse_start_frame = 1; - Timestamp reverse_start_box_time = - input_frames_packets_[reverse_start_frame].Timestamp() + 1000; - - auto ra_boxes = CreateRandomAccessTrackingBoxList( - {start_box_time, reverse_start_box_time}, {end_box_time, start_box_time}); - - Packet ra_packet = Adopt(ra_boxes.release()).At(start_box_time); - Packet start_packet = Adopt(start_box_list.release()).At(start_box_time); - - // Start running the ordinary graph, verify random access produce same result - // as normal tracking. - MP_EXPECT_OK(graph_.StartRun(side_packets)); - MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_packet)); - for (auto frame_packet : input_frames_packets_) { - MP_EXPECT_OK( - graph_.AddPacketToInputStream("image_cpu_frames", frame_packet)); - Packet track_time_packet = Adopt(new int(0)).At(frame_packet.Timestamp()); - MP_EXPECT_OK( - graph_.AddPacketToInputStream("track_time", track_time_packet)); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - } - MP_EXPECT_OK(graph_.AddPacketToInputStream("ra_track", ra_packet)); - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); - - EXPECT_EQ(input_frames_packets_.size(), output_packets_.size()); - const TimedBoxProtoList tracking_result = - output_packets_[end_frame].Get(); - EXPECT_EQ(1, tracking_result.box_size()); - - // Should have 1 random access packet. - EXPECT_EQ(1, random_access_results_packets_.size()); - const TimedBoxProtoList& ra_result = - random_access_results_packets_[0].Get(); - // Two box tracking results. One for comparison with normal tracking. The - // other for reverse random access tracking. - EXPECT_EQ(2, ra_result.box_size()); - - // Check if randan access tracking has same result with normal tracking. - ExpectQuadNear(tracking_result.box(0), ra_result.box(0)); - ExpectQuadAtFrame(ra_result.box(0), end_frame - start_frame, - kImageAspectRatio, false); - ExpectQuadAtFrame(ra_result.box(1), start_frame - reverse_start_frame - 1, - kImageAspectRatio, false); - - // Clear output and ra result packet vector before test parallel graph. - TearDown(); - - // Start running the parallel graph, verify random access produce same result - // as normal tracking. - MP_EXPECT_OK(parallel_graph_.StartRun(side_packets)); - MP_EXPECT_OK( - parallel_graph_.AddPacketToInputStream("start_pos", start_packet)); - for (auto frame_packet : input_frames_packets_) { - MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames", - frame_packet)); - MP_EXPECT_OK(parallel_graph_.WaitUntilIdle()); - } - MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet)); - MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams()); - MP_EXPECT_OK(parallel_graph_.WaitUntilDone()); - - EXPECT_EQ(input_frames_packets_.size(), output_packets_.size()); - const TimedBoxProtoList parallel_tracking_result = - output_packets_[end_frame].Get(); - EXPECT_EQ(1, parallel_tracking_result.box_size()); - - // should have only 1 random access - EXPECT_EQ(1, random_access_results_packets_.size()); - const TimedBoxProtoList& parallel_ra_result = - random_access_results_packets_[0].Get(); - EXPECT_EQ(2, parallel_ra_result.box_size()); - - // Check if randan access tracking has same result with normal tracking. - ExpectQuadNear(parallel_tracking_result.box(0), parallel_ra_result.box(0)); - ExpectQuadAtFrame(parallel_ra_result.box(0), end_frame - start_frame, - kImageAspectRatio, false); - ExpectQuadAtFrame(parallel_ra_result.box(1), - start_frame - reverse_start_frame - 1, kImageAspectRatio, - false); -} - -// Tests what happens when random access request timestamps are -// outside of cache. -TEST_F(TrackingGraphTest, TestRandomAccessTrackingTimestamps) { - // Create input side packets. - std::map side_packets; - side_packets.insert(std::make_pair("analysis_downsample_factor", - mediapipe::MakePacket(1.0f))); - CalculatorOptions calculator_options; - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->mutable_tracker_options() - ->mutable_track_step_options() - ->set_tracking_degrees( - TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE); - // We intentionally don't cache all frames, to see what happens when - // random access tracking request time falls outside cache range. - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->set_streaming_track_data_cache_size(input_frames_packets_.size() - 1); - side_packets.insert(std::make_pair( - "calculator_options", - mediapipe::MakePacket(calculator_options))); - - // Set up random access boxes - const int num_frames = input_frames_packets_.size(); - const int64 usec_in_sec = 1000000; - std::vector start_timestamps{ - input_frames_packets_[0].Timestamp() - usec_in_sec, // forward - input_frames_packets_[0].Timestamp(), // forward - input_frames_packets_[1].Timestamp(), // forward - input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec, // fwd - input_frames_packets_[0].Timestamp(), // backward - input_frames_packets_[num_frames - 1].Timestamp(), // backward - input_frames_packets_[num_frames - 1].Timestamp(), // backward - input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec // back - }; - std::vector end_timestamps{ - input_frames_packets_[num_frames - 1].Timestamp(), - input_frames_packets_[num_frames - 1].Timestamp(), - input_frames_packets_[num_frames - 1].Timestamp() + usec_in_sec, - input_frames_packets_[num_frames - 1].Timestamp() + 2 * usec_in_sec, - input_frames_packets_[0].Timestamp() - usec_in_sec, - input_frames_packets_[0].Timestamp(), - input_frames_packets_[0].Timestamp() - usec_in_sec, - input_frames_packets_[1].Timestamp()}; - auto ra_boxes = - CreateRandomAccessTrackingBoxList(start_timestamps, end_timestamps); - Packet ra_packet = - Adopt(ra_boxes.release()).At(input_frames_packets_[0].Timestamp()); - - // Run the graph and check if the outside-cache request have no results. - // Start running the parallel graph, verify random access produce same result - // as normal tracking. - MP_EXPECT_OK(parallel_graph_.StartRun(side_packets)); - for (auto frame_packet : input_frames_packets_) { - MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("image_cpu_frames", - frame_packet)); - MP_EXPECT_OK(parallel_graph_.WaitUntilIdle()); - } - MP_EXPECT_OK(parallel_graph_.AddPacketToInputStream("ra_track", ra_packet)); - MP_EXPECT_OK(parallel_graph_.CloseAllInputStreams()); - MP_EXPECT_OK(parallel_graph_.WaitUntilDone()); - - // should have 1 random access packet with 0 result boxes - EXPECT_EQ(1, random_access_results_packets_.size()); - const auto& ra_returned_boxes = - random_access_results_packets_[0].Get(); - const int num_returned_ra_boxes = ra_returned_boxes.box_size(); - EXPECT_EQ(0, num_returned_ra_boxes); -} - -TEST_F(TrackingGraphTest, TestTransitionFramesForReacquisition) { - // Create input side packets. - std::map side_packets; - side_packets.insert(std::make_pair("analysis_downsample_factor", - mediapipe::MakePacket(1.0f))); - CalculatorOptions calculator_options; - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->mutable_tracker_options() - ->mutable_track_step_options() - ->set_tracking_degrees( - TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE); - constexpr int kTransitionFrames = 3; - calculator_options.MutableExtension(BoxTrackerCalculatorOptions::ext) - ->set_start_pos_transition_frames(kTransitionFrames); - - side_packets.insert(std::make_pair( - "calculator_options", - mediapipe::MakePacket(calculator_options))); - - Timestamp start_box_time = input_frames_packets_[0].Timestamp(); - // Box id 0 use quad tracking with 8DoF homography transform. - // Box id 1 use quad tracking with 6DoF perspective transform. - // Box id 2 use box tracking with 4DoF similarity transform. - std::vector is_quad_tracking{true, true, false}; - std::vector is_pnp_tracking{false, true, false}; - std::vector is_reacquisition{true, true, true}; - auto start_box_list = MakeBoxList(start_box_time, is_quad_tracking, - is_pnp_tracking, is_reacquisition); - Packet start_pos_packet = Adopt(start_box_list.release()).At(start_box_time); - - // Setting box pos restart from initial position (frame 0's position). - constexpr int kRestartFrame = 3; - Timestamp restart_box_time = input_frames_packets_[kRestartFrame].Timestamp(); - auto restart_box_list = MakeBoxList(restart_box_time, is_quad_tracking, - is_pnp_tracking, is_reacquisition); - Packet restart_pos_packet = - Adopt(restart_box_list.release()).At(restart_box_time); - MP_EXPECT_OK(graph_.StartRun(side_packets)); - MP_EXPECT_OK(graph_.AddPacketToInputStream("start_pos", start_pos_packet)); - - for (int j = 0; j < input_frames_packets_.size(); ++j) { - // Add TRACK_TIME stream queries in between 2 frames. - if (j > 0) { - Timestamp track_time = Timestamp((j - 0.5f) * kFrameIntervalUs); - LOG(INFO) << track_time.Value(); - Packet track_time_packet = Adopt(new Timestamp).At(track_time); - MP_EXPECT_OK( - graph_.AddPacketToInputStream("track_time", track_time_packet)); - } - - MP_EXPECT_OK(graph_.AddPacketToInputStream("image_cpu_frames", - input_frames_packets_[j])); - Packet track_time_packet = - Adopt(new int(0)).At(input_frames_packets_[j].Timestamp()); - MP_EXPECT_OK( - graph_.AddPacketToInputStream("track_time", track_time_packet)); - MP_EXPECT_OK(graph_.WaitUntilIdle()); - - if (j == kRestartFrame) { - MP_EXPECT_OK( - graph_.AddPacketToInputStream("restart_pos", restart_pos_packet)); - } - } - - MP_EXPECT_OK(graph_.CloseAllInputStreams()); - MP_EXPECT_OK(graph_.WaitUntilDone()); - - EXPECT_EQ(input_frames_packets_.size() * 2 - 1, output_packets_.size()); - for (int i = 0; i < output_packets_.size(); ++i) { - const TimedBoxProtoList& boxes = - output_packets_[i].Get(); - EXPECT_EQ(is_quad_tracking.size(), boxes.box_size()); - float frame_id = i / 2.0f; - float expected_frame_id; - if (frame_id <= kRestartFrame) { - // before transition - expected_frame_id = frame_id; - } else { - float transition_frames = frame_id - kRestartFrame; - if (transition_frames <= kTransitionFrames) { - // transitioning. - expected_frame_id = - kRestartFrame - - transition_frames / kTransitionFrames * kRestartFrame + - transition_frames; - } else { - // after transition. - expected_frame_id = transition_frames; - } - } - - for (int j = 0; j < boxes.box_size(); ++j) { - const TimedBoxProto& box = boxes.box(j); - if (is_quad_tracking[box.id()]) { - ExpectQuadAtFrame(box, expected_frame_id, - is_pnp_tracking[box.id()] ? kImageAspectRatio : -1.0f, - is_reacquisition[box.id()]); - } else { - ExpectBoxAtFrame(box, expected_frame_id, is_reacquisition[box.id()]); - } - } - } -} - -// TODO: Add test for reacquisition. - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc b/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc deleted file mode 100644 index cf00da1f7..000000000 --- a/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/base/macros.h" -#include "absl/synchronization/mutex.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/motion/optical_flow_field.h" -#include "mediapipe/framework/port/opencv_video_inc.h" - -namespace mediapipe { -namespace { - -// Checks that img1 and img2 have the same dimensions. -bool ImageSizesMatch(const ImageFrame& img1, const ImageFrame& img2) { - return (img1.Width() == img2.Width()) && (img1.Height() == img2.Height()); -} - -// Converts an RGB image to grayscale. -cv::Mat ConvertToGrayscale(const cv::Mat& image) { - if (image.channels() == 1) { - return image; - } - cv::Mat gray; - cv::cvtColor(image, gray, cv::COLOR_RGB2GRAY); - return gray; -} - -} // namespace - -// Calls OpenCV's DenseOpticalFlow to compute the optical flow between a pair of -// image frames. The calculator can output forward flow fields (optical flow -// from the first frame to the second frame), backward flow fields (optical flow -// from the second frame to the first frame), or both, depending on the tag of -// the specified output streams. Note that the timestamp of the output optical -// flow is always tied to the input timestamp. Be aware of the different -// meanings of the timestamp between the forward and the backward optical flows -// if the calculator outputs both. -// -// If the "max_in_flight" field is set to any value greater than 1, it will -// enable the calculator to process multiple inputs in parallel. The output -// packets will be automatically ordered by timestamp before they are passed -// along to downstream calculators. -// -// Inputs: -// FIRST_FRAME: An ImageFrame in either SRGB or GRAY8 format. -// SECOND_FRAME: An ImageFrame in either SRGB or GRAY8 format. -// Outputs: -// FORWARD_FLOW: The OpticalFlowField from the first frame to the second -// frame, output at the input timestamp. -// BACKWARD_FLOW: The OpticalFlowField from the second frame to the first -// frame, output at the input timestamp. -// Example config: -// node { -// calculator: "Tvl1OpticalFlowCalculator" -// input_stream: "FIRST_FRAME:first_frames" -// input_stream: "SECOND_FRAME:second_frames" -// output_stream: "FORWARD_FLOW:forward_flow" -// output_stream: "BACKWARD_FLOW:backward_flow" -// max_in_flight: 10 -// } -// num_threads: 10 -class Tvl1OpticalFlowCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::Status CalculateOpticalFlow(const ImageFrame& current_frame, - const ImageFrame& next_frame, - OpticalFlowField* flow); - bool forward_requested_ = false; - bool backward_requested_ = false; - // Stores the idle DenseOpticalFlow objects. - // cv::DenseOpticalFlow is not thread-safe. Invoking multiple - // DenseOpticalFlow::calc() in parallel may lead to memory corruption or - // memory leak. - std::list> tvl1_computers_ - ABSL_GUARDED_BY(mutex_); - absl::Mutex mutex_; -}; - -absl::Status Tvl1OpticalFlowCalculator::GetContract(CalculatorContract* cc) { - if (!cc->Inputs().HasTag("FIRST_FRAME") || - !cc->Inputs().HasTag("SECOND_FRAME")) { - return absl::InvalidArgumentError( - "Missing required input streams. Both FIRST_FRAME and SECOND_FRAME " - "must be specified."); - } - cc->Inputs().Tag("FIRST_FRAME").Set(); - cc->Inputs().Tag("SECOND_FRAME").Set(); - if (cc->Outputs().HasTag("FORWARD_FLOW")) { - cc->Outputs().Tag("FORWARD_FLOW").Set(); - } - if (cc->Outputs().HasTag("BACKWARD_FLOW")) { - cc->Outputs().Tag("BACKWARD_FLOW").Set(); - } - return absl::OkStatus(); -} - -absl::Status Tvl1OpticalFlowCalculator::Open(CalculatorContext* cc) { - { - absl::MutexLock lock(&mutex_); - tvl1_computers_.emplace_back(cv::createOptFlow_DualTVL1()); - } - if (cc->Outputs().HasTag("FORWARD_FLOW")) { - forward_requested_ = true; - } - if (cc->Outputs().HasTag("BACKWARD_FLOW")) { - backward_requested_ = true; - } - - return absl::OkStatus(); -} - -absl::Status Tvl1OpticalFlowCalculator::Process(CalculatorContext* cc) { - const ImageFrame& first_frame = - cc->Inputs().Tag("FIRST_FRAME").Value().Get(); - const ImageFrame& second_frame = - cc->Inputs().Tag("SECOND_FRAME").Value().Get(); - if (forward_requested_) { - auto forward_optical_flow_field = absl::make_unique(); - MP_RETURN_IF_ERROR(CalculateOpticalFlow(first_frame, second_frame, - forward_optical_flow_field.get())); - cc->Outputs() - .Tag("FORWARD_FLOW") - .Add(forward_optical_flow_field.release(), cc->InputTimestamp()); - } - if (backward_requested_) { - auto backward_optical_flow_field = absl::make_unique(); - MP_RETURN_IF_ERROR(CalculateOpticalFlow(second_frame, first_frame, - backward_optical_flow_field.get())); - cc->Outputs() - .Tag("BACKWARD_FLOW") - .Add(backward_optical_flow_field.release(), cc->InputTimestamp()); - } - return absl::OkStatus(); -} - -absl::Status Tvl1OpticalFlowCalculator::CalculateOpticalFlow( - const ImageFrame& current_frame, const ImageFrame& next_frame, - OpticalFlowField* flow) { - CHECK(flow); - if (!ImageSizesMatch(current_frame, next_frame)) { - return tool::StatusInvalid("Images are different sizes."); - } - const cv::Mat& first = ConvertToGrayscale(formats::MatView(¤t_frame)); - const cv::Mat& second = ConvertToGrayscale(formats::MatView(&next_frame)); - - // Tries getting an idle DenseOpticalFlow object from the cache. If not, - // creates a new DenseOpticalFlow. - cv::Ptr tvl1_computer; - { - absl::MutexLock lock(&mutex_); - if (!tvl1_computers_.empty()) { - std::swap(tvl1_computer, tvl1_computers_.front()); - tvl1_computers_.pop_front(); - } - } - if (tvl1_computer.empty()) { - tvl1_computer = cv::createOptFlow_DualTVL1(); - } - - flow->Allocate(first.cols, first.rows); - cv::Mat cv_flow(flow->mutable_flow_data()); - tvl1_computer->calc(first, second, cv_flow); - CHECK_EQ(flow->mutable_flow_data().data, cv_flow.data); - // Inserts the idle DenseOpticalFlow object back to the cache for reuse. - { - absl::MutexLock lock(&mutex_); - tvl1_computers_.push_back(tvl1_computer); - } - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(Tvl1OpticalFlowCalculator); - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/tvl1_optical_flow_calculator_test.cc b/mediapipe/calculators/video/tvl1_optical_flow_calculator_test.cc deleted file mode 100644 index c9d30b73d..000000000 --- a/mediapipe/calculators/video/tvl1_optical_flow_calculator_test.cc +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/motion/optical_flow_field.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { - -namespace { -void AddInputPackets(int num_packets, CalculatorGraph* graph) { - int width = 127; - int height = 227; - Packet packet1 = MakePacket(ImageFormat::SRGB, width, height); - Packet packet2 = MakePacket(ImageFormat::SRGB, width, height); - cv::Mat mat1 = formats::MatView(&(packet1.Get())); - cv::Mat mat2 = formats::MatView(&(packet2.Get())); - for (int r = 0; r < mat1.rows; ++r) { - for (int c = 0; c < mat1.cols; ++c) { - cv::Vec3b& color1 = mat1.at(r, c); - color1[0] = r + 3; - color1[1] = r + 3; - color1[2] = 0; - cv::Vec3b& color2 = mat2.at(r, c); - color2[0] = r; - color2[1] = r; - color2[2] = 0; - } - } - - for (int i = 0; i < num_packets; ++i) { - MP_ASSERT_OK(graph->AddPacketToInputStream("first_frames", - packet1.At(Timestamp(i)))); - MP_ASSERT_OK(graph->AddPacketToInputStream("second_frames", - packet2.At(Timestamp(i)))); - } - MP_ASSERT_OK(graph->CloseAllInputStreams()); -} - -void RunTest(int num_input_packets, int max_in_flight) { - CalculatorGraphConfig config = ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "first_frames" - input_stream: "second_frames" - node { - calculator: "Tvl1OpticalFlowCalculator" - input_stream: "FIRST_FRAME:first_frames" - input_stream: "SECOND_FRAME:second_frames" - output_stream: "FORWARD_FLOW:forward_flow" - output_stream: "BACKWARD_FLOW:backward_flow" - max_in_flight: $0 - } - num_threads: $0 - )", - max_in_flight)); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - StatusOrPoller status_or_poller1 = - graph.AddOutputStreamPoller("forward_flow"); - ASSERT_TRUE(status_or_poller1.ok()); - OutputStreamPoller poller1 = std::move(status_or_poller1.value()); - StatusOrPoller status_or_poller2 = - graph.AddOutputStreamPoller("backward_flow"); - ASSERT_TRUE(status_or_poller2.ok()); - OutputStreamPoller poller2 = std::move(status_or_poller2.value()); - - MP_ASSERT_OK(graph.StartRun({})); - AddInputPackets(num_input_packets, &graph); - Packet packet; - std::vector forward_optical_flow_packets; - while (poller1.Next(&packet)) { - forward_optical_flow_packets.emplace_back(packet); - } - std::vector backward_optical_flow_packets; - while (poller2.Next(&packet)) { - backward_optical_flow_packets.emplace_back(packet); - } - MP_ASSERT_OK(graph.WaitUntilDone()); - EXPECT_EQ(num_input_packets, forward_optical_flow_packets.size()); - - int count = 0; - for (const Packet& packet : forward_optical_flow_packets) { - cv::Scalar average = cv::mean(packet.Get().flow_data()); - EXPECT_NEAR(average[0], 0.0, 0.5) << "Actual mean_dx = " << average[0]; - EXPECT_NEAR(average[1], 3.0, 0.5) << "Actual mean_dy = " << average[1]; - EXPECT_EQ(count++, packet.Timestamp().Value()); - } - EXPECT_EQ(num_input_packets, backward_optical_flow_packets.size()); - count = 0; - for (const Packet& packet : backward_optical_flow_packets) { - cv::Scalar average = cv::mean(packet.Get().flow_data()); - EXPECT_NEAR(average[0], 0.0, 0.5) << "Actual mean_dx = " << average[0]; - EXPECT_NEAR(average[1], -3.0, 0.5) << "Actual mean_dy = " << average[1]; - EXPECT_EQ(count++, packet.Timestamp().Value()); - } -} - -TEST(Tvl1OpticalFlowCalculatorTest, TestSequentialExecution) { - RunTest(/*num_input_packets=*/2, /*max_in_flight=*/1); -} - -TEST(Tvl1OpticalFlowCalculatorTest, TestParallelExecution) { - RunTest(/*num_input_packets=*/20, /*max_in_flight=*/10); -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/calculators/video/video_pre_stream_calculator.cc b/mediapipe/calculators/video/video_pre_stream_calculator.cc deleted file mode 100644 index ab9cd22a4..000000000 --- a/mediapipe/calculators/video/video_pre_stream_calculator.cc +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/video/video_pre_stream_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/video_stream_header.h" - -namespace mediapipe { - -// Sets up VideoHeader based on the 1st ImageFrame and emits it with timestamp -// PreStream. Note that this calculator only fills in format, width, and height, -// i.e. frame_rate and duration will not be filled, unless: -// 1) an existing VideoHeader is provided at PreStream(). In such case, the -// frame_rate and duration, if they exist, will be copied from the existing -// VideoHeader. -// 2) you specify frame_rate and duration through the options. In this case, the -// options will overwrite the existing VideoHeader if it is available. -// -// Example config: -// node { -// calculator: "VideoPreStreamCalculator" -// input_stream: "FRAME:cropped_frames" -// input_stream: "VIDEO_PRESTREAM:original_video_header" -// output_stream: "cropped_frames_video_header" -// } -// -// or -// -// node { -// calculator: "VideoPreStreamCalculator" -// input_stream: "cropped_frames" -// output_stream: "video_header" -// } -class VideoPreStreamCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - absl::Status Open(CalculatorContext* cc) override; - absl::Status Process(CalculatorContext* cc) override; - - private: - absl::Status ProcessWithFrameRateInPreStream(CalculatorContext* cc); - absl::Status ProcessWithFrameRateInOptions(CalculatorContext* cc); - - std::unique_ptr header_; - bool frame_rate_in_prestream_ = false; - bool emitted_ = false; -}; - -REGISTER_CALCULATOR(VideoPreStreamCalculator); - -absl::Status VideoPreStreamCalculator::GetContract(CalculatorContract* cc) { - if (!cc->Inputs().UsesTags()) { - cc->Inputs().Index(0).Set(); - } else { - cc->Inputs().Tag("FRAME").Set(); - cc->Inputs().Tag("VIDEO_PRESTREAM").Set(); - } - cc->Outputs().Index(0).Set(); - return absl::OkStatus(); -} - -absl::Status VideoPreStreamCalculator::Open(CalculatorContext* cc) { - frame_rate_in_prestream_ = cc->Inputs().UsesTags() && - cc->Inputs().HasTag("FRAME") && - cc->Inputs().HasTag("VIDEO_PRESTREAM"); - header_ = absl::make_unique(); - return absl::OkStatus(); -} -absl::Status VideoPreStreamCalculator::ProcessWithFrameRateInPreStream( - CalculatorContext* cc) { - cc->GetCounter("ProcessWithFrameRateInPreStream")->Increment(); - if (cc->InputTimestamp() == Timestamp::PreStream()) { - RET_CHECK(cc->Inputs().Tag("FRAME").IsEmpty()); - RET_CHECK(!cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty()); - *header_ = cc->Inputs().Tag("VIDEO_PRESTREAM").Get(); - RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero"; - } else { - RET_CHECK(cc->Inputs().Tag("VIDEO_PRESTREAM").IsEmpty()) - << "Packet on VIDEO_PRESTREAM must come in at Timestamp::PreStream()."; - RET_CHECK(!cc->Inputs().Tag("FRAME").IsEmpty()); - const auto& frame = cc->Inputs().Tag("FRAME").Get(); - header_->format = frame.Format(); - header_->width = frame.Width(); - header_->height = frame.Height(); - RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero"; - cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream()); - emitted_ = true; - } - return absl::OkStatus(); -} - -absl::Status VideoPreStreamCalculator::Process(CalculatorContext* cc) { - cc->GetCounter("Process")->Increment(); - if (emitted_) { - return absl::OkStatus(); - } - if (frame_rate_in_prestream_) { - return ProcessWithFrameRateInPreStream(cc); - } else { - return ProcessWithFrameRateInOptions(cc); - } -} - -absl::Status VideoPreStreamCalculator::ProcessWithFrameRateInOptions( - CalculatorContext* cc) { - cc->GetCounter("ProcessWithFrameRateInOptions")->Increment(); - RET_CHECK_NE(cc->InputTimestamp(), Timestamp::PreStream()); - const auto& frame = cc->Inputs().Index(0).Get(); - header_->format = frame.Format(); - header_->width = frame.Width(); - header_->height = frame.Height(); - const auto& options = cc->Options(); - if (options.fps().has_value()) { - header_->frame_rate = options.fps().value(); - } else if (options.fps().has_ratio()) { - const VideoPreStreamCalculatorOptions::Fps::Rational32& ratio = - options.fps().ratio(); - if (ratio.numerator() > 0 && ratio.denominator() > 0) { - header_->frame_rate = - static_cast(ratio.numerator()) / ratio.denominator(); - } - } - RET_CHECK_NE(header_->frame_rate, 0.0) << "frame rate should be non-zero"; - cc->Outputs().Index(0).Add(header_.release(), Timestamp::PreStream()); - emitted_ = true; - return absl::OkStatus(); -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/video/video_pre_stream_calculator.proto b/mediapipe/calculators/video/video_pre_stream_calculator.proto deleted file mode 100644 index 8e87ce995..000000000 --- a/mediapipe/calculators/video/video_pre_stream_calculator.proto +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe; - -import "mediapipe/framework/calculator.proto"; - -message VideoPreStreamCalculatorOptions { - extend CalculatorOptions { - optional VideoPreStreamCalculatorOptions ext = 151386123; - } - - // An arbitrary number of frames per second. - // Prefer the StandardFps enum to store industry-standard, safe FPS values. - message Fps { - // The possibly approximated value of the frame rate, in frames per second. - // Unsafe to use in accurate computations because prone to rounding errors. - // For example, the 23.976 FPS value has no exact representation as a - // double. - optional double value = 1; - - message Rational32 { - optional int32 numerator = 1; - optional int32 denominator = 2; - } - // The exact value of the frame rate, as a rational number. - optional Rational32 ratio = 2; - } - optional Fps fps = 1; -} diff --git a/mediapipe/calculators/video/video_pre_stream_calculator_test.cc b/mediapipe/calculators/video/video_pre_stream_calculator_test.cc deleted file mode 100644 index 3f864ff88..000000000 --- a/mediapipe/calculators/video/video_pre_stream_calculator_test.cc +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInOptions) { - auto config = ParseTextProtoOrDie(R"pb( - input_stream: "input" - node { - calculator: "VideoPreStreamCalculator" - input_stream: "input" - output_stream: "output" - options { - [mediapipe.VideoPreStreamCalculatorOptions.ext] { fps { value: 3 } } - } - })pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - auto poller_status = graph.AddOutputStreamPoller("output"); - MP_ASSERT_OK(poller_status.status()); - OutputStreamPoller& poller = poller_status.value(); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input", - Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0)))); - - // It is *not* VideoPreStreamCalculator's job to detect errors in an - // ImageFrame stream. It just waits for the 1st ImageFrame, extracts info for - // VideoHeader, and emits it. Thus, the following is fine. - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input", - Adopt(new ImageFrame(ImageFormat::SRGBA, 3, 4)).At(Timestamp(1)))); - - MP_ASSERT_OK(graph.CloseInputStream("input")); - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - const auto& video_header = packet.Get(); - EXPECT_EQ(video_header.format, ImageFormat::SRGB); - EXPECT_EQ(video_header.width, 1); - EXPECT_EQ(video_header.height, 2); - EXPECT_EQ(video_header.frame_rate, 3); - EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream()); - ASSERT_FALSE(poller.Next(&packet)); - MP_EXPECT_OK(graph.WaitUntilDone()); -} - -TEST(VideoPreStreamCalculatorTest, ProcessesWithFrameRateInPreStream) { - auto config = ParseTextProtoOrDie(R"pb( - input_stream: "frame" - input_stream: "input_header" - node { - calculator: "VideoPreStreamCalculator" - input_stream: "FRAME:frame" - input_stream: "VIDEO_PRESTREAM:input_header" - output_stream: "output_header" - })pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - auto poller_status = graph.AddOutputStreamPoller("output_header"); - MP_ASSERT_OK(poller_status.status()); - OutputStreamPoller& poller = poller_status.value(); - MP_ASSERT_OK(graph.StartRun({})); - auto input_header = absl::make_unique(); - input_header->frame_rate = 3.0; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_header", - Adopt(input_header.release()).At(Timestamp::PreStream()))); - MP_ASSERT_OK(graph.CloseInputStream("input_header")); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "frame", - Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("frame")); - Packet packet; - ASSERT_TRUE(poller.Next(&packet)); - const auto& output_header = packet.Get(); - EXPECT_EQ(output_header.format, ImageFormat::SRGB); - EXPECT_EQ(output_header.width, 1); - EXPECT_EQ(output_header.height, 2); - EXPECT_EQ(output_header.frame_rate, 3.0); - EXPECT_EQ(packet.Timestamp(), Timestamp::PreStream()); - ASSERT_FALSE(poller.Next(&packet)); - MP_EXPECT_OK(graph.WaitUntilDone()); -} - -TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInOptions) { - auto config = ParseTextProtoOrDie(R"pb( - input_stream: "frame" - node { - calculator: "VideoPreStreamCalculator" - input_stream: "frame" - output_stream: "output_header" - })pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "frame", - Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("frame")); - absl::Status status = graph.WaitUntilDone(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("frame rate should be non-zero")); -} - -// Input header missing. -TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream1) { - auto config = ParseTextProtoOrDie(R"pb( - input_stream: "frame" - input_stream: "input_header" - node { - calculator: "VideoPreStreamCalculator" - input_stream: "FRAME:frame" - input_stream: "VIDEO_PRESTREAM:input_header" - output_stream: "output_header" - } - )pb"); - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - MP_ASSERT_OK(graph.StartRun({})); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "frame", - Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("frame")); - MP_ASSERT_OK(graph.CloseInputStream("input_header")); - absl::Status status = graph.WaitUntilDone(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("frame rate should be non-zero")); -} - -// Input header not at prestream (before, with, and after frame data). -TEST(VideoPreStreamCalculatorTest, FailsWithoutFrameRateInPreStream2) { - auto config = ParseTextProtoOrDie(R"pb( - input_stream: "frame" - input_stream: "input_header" - node { - calculator: "VideoPreStreamCalculator" - input_stream: "FRAME:frame" - input_stream: "VIDEO_PRESTREAM:input_header" - output_stream: "output_header" - } - )pb"); - - for (int64 timestamp = -1; timestamp < 2; ++timestamp) { - CalculatorGraph graph; - MP_ASSERT_OK(graph.Initialize(config)); - MP_ASSERT_OK(graph.StartRun({})); - auto input_header = absl::make_unique(); - input_header->frame_rate = 3.0; - MP_ASSERT_OK(graph.AddPacketToInputStream( - "input_header", - Adopt(input_header.release()).At(Timestamp(timestamp)))); - MP_ASSERT_OK(graph.CloseInputStream("input_header")); - MP_ASSERT_OK(graph.AddPacketToInputStream( - "frame", - Adopt(new ImageFrame(ImageFormat::SRGB, 1, 2)).At(Timestamp(0)))); - MP_ASSERT_OK(graph.CloseInputStream("frame")); - absl::Status status = graph.WaitUntilDone(); - EXPECT_FALSE(status.ok()); - } -} - -} // namespace -} // namespace mediapipe diff --git a/mediapipe/docs/README.md b/mediapipe/docs/README.md deleted file mode 100644 index 70c3d2723..000000000 --- a/mediapipe/docs/README.md +++ /dev/null @@ -1,3 +0,0 @@ -This directory contains legacy markdown docs referenced in external sites and blog posts, and the docs have messages to redirect users to the corresponding up-to-date docs in other locations. - -Source files of the update-to-date docs are in `docs` directly under root. diff --git a/mediapipe/docs/autoflip.md b/mediapipe/docs/autoflip.md deleted file mode 100644 index 4d7367810..000000000 --- a/mediapipe/docs/autoflip.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[AutoFlip: Saliency-aware Video Cropping](https://google.github.io/mediapipe/solutions/autoflip) diff --git a/mediapipe/docs/face_detection_desktop.md b/mediapipe/docs/face_detection_desktop.md deleted file mode 100644 index 8377e8df1..000000000 --- a/mediapipe/docs/face_detection_desktop.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Face Detection](https://google.github.io/mediapipe/solutions/face_detection) diff --git a/mediapipe/docs/face_detection_mobile_gpu.md b/mediapipe/docs/face_detection_mobile_gpu.md deleted file mode 100644 index 8377e8df1..000000000 --- a/mediapipe/docs/face_detection_mobile_gpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Face Detection](https://google.github.io/mediapipe/solutions/face_detection) diff --git a/mediapipe/docs/hair_segmentation_mobile_gpu.md b/mediapipe/docs/hair_segmentation_mobile_gpu.md deleted file mode 100644 index 43116a4f6..000000000 --- a/mediapipe/docs/hair_segmentation_mobile_gpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) diff --git a/mediapipe/docs/hand_tracking_desktop.md b/mediapipe/docs/hand_tracking_desktop.md deleted file mode 100644 index 02bb1312c..000000000 --- a/mediapipe/docs/hand_tracking_desktop.md +++ /dev/null @@ -1 +0,0 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/hand_tracking_mobile_gpu.md b/mediapipe/docs/hand_tracking_mobile_gpu.md deleted file mode 100644 index 02bb1312c..000000000 --- a/mediapipe/docs/hand_tracking_mobile_gpu.md +++ /dev/null @@ -1 +0,0 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md deleted file mode 100644 index 02bb1312c..000000000 --- a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md +++ /dev/null @@ -1 +0,0 @@ -Content moved to [MediaPipe Hands](https://google.github.io/mediapipe/solutions/hands) diff --git a/mediapipe/docs/object_detection_desktop.md b/mediapipe/docs/object_detection_desktop.md deleted file mode 100644 index 2e565cefd..000000000 --- a/mediapipe/docs/object_detection_desktop.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Object Detection](https://google.github.io/mediapipe/solutions/object_detection) diff --git a/mediapipe/docs/object_detection_mobile_gpu.md b/mediapipe/docs/object_detection_mobile_gpu.md deleted file mode 100644 index 2e565cefd..000000000 --- a/mediapipe/docs/object_detection_mobile_gpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Object Detection](https://google.github.io/mediapipe/solutions/object_detection) diff --git a/mediapipe/docs/object_tracking_mobile_gpu.md b/mediapipe/docs/object_tracking_mobile_gpu.md deleted file mode 100644 index c74d942f6..000000000 --- a/mediapipe/docs/object_tracking_mobile_gpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) diff --git a/mediapipe/docs/objectron_mobile_gpu.md b/mediapipe/docs/objectron_mobile_gpu.md deleted file mode 100644 index 231fc512c..000000000 --- a/mediapipe/docs/objectron_mobile_gpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe Objectron](https://google.github.io/mediapipe/solutions/objectron) diff --git a/mediapipe/docs/template_matching_mobile_cpu.md b/mediapipe/docs/template_matching_mobile_cpu.md deleted file mode 100644 index 02150175c..000000000 --- a/mediapipe/docs/template_matching_mobile_cpu.md +++ /dev/null @@ -1,2 +0,0 @@ -Content moved to -[MediaPipe KNIFT](https://google.github.io/mediapipe/solutions/knift) diff --git a/mediapipe/examples/__init__.py b/mediapipe/examples/__init__.py deleted file mode 100644 index 6db73bc52..000000000 --- a/mediapipe/examples/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Copyright 2019 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/mediapipe/examples/android/README.md b/mediapipe/examples/android/README.md deleted file mode 100644 index bc32c24da..000000000 --- a/mediapipe/examples/android/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains MediaPipe example applications for Android. Please see [Solutions](https://solutions.mediapipe.dev) for details. diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml deleted file mode 100644 index f7218c97c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD deleted file mode 100644 index ae4652dba..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/BUILD +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -# Basic library common across example apps. -android_library( - name = "basic_lib", - srcs = glob(["*.java"]), - manifest = "AndroidManifest.xml", - resource_files = glob(["res/**"]), - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper", - "//mediapipe/java/com/google/mediapipe/components:android_components", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "//mediapipe/java/com/google/mediapipe/glutil", - "//third_party:androidx_appcompat", - "//third_party:androidx_constraint_layout", - "//third_party:opencv", - "@maven//:androidx_concurrent_concurrent_futures", - "@maven//:com_google_guava_guava", - ], -) - -# Manifest common across example apps. -exports_files( - srcs = ["AndroidManifest.xml"], -) - -# Native dependencies to perform edge detection in the Hello World example. -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/edge_detection:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -# Converts the .so cc_binary into a cc_library, to be consumed in an android_binary. -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -# Hello World example app. -android_binary( - name = "helloworld", - assets = [ - "//mediapipe/graphs/edge_detection:mobile_gpu.binarypb", - ], - assets_dir = "", - manifest = "AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.basic", - "appName": "Hello World", - "mainActivity": ".MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":basic_lib", - ":mediapipe_jni_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java deleted file mode 100644 index 952132cdf..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/MainActivity.java +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.basic; - -import android.content.pm.ApplicationInfo; -import android.content.pm.PackageManager; -import android.content.pm.PackageManager.NameNotFoundException; -import android.graphics.SurfaceTexture; -import android.os.Bundle; -import androidx.appcompat.app.AppCompatActivity; -import android.util.Log; -import android.util.Size; -import android.view.SurfaceHolder; -import android.view.SurfaceView; -import android.view.View; -import android.view.ViewGroup; -import com.google.mediapipe.components.CameraHelper; -import com.google.mediapipe.components.CameraXPreviewHelper; -import com.google.mediapipe.components.ExternalTextureConverter; -import com.google.mediapipe.components.FrameProcessor; -import com.google.mediapipe.components.PermissionHelper; -import com.google.mediapipe.framework.AndroidAssetUtil; -import com.google.mediapipe.glutil.EglManager; - -/** Main activity of MediaPipe basic app. */ -public class MainActivity extends AppCompatActivity { - private static final String TAG = "MainActivity"; - - // Flips the camera-preview frames vertically by default, before sending them into FrameProcessor - // to be processed in a MediaPipe graph, and flips the processed frames back when they are - // displayed. This maybe needed because OpenGL represents images assuming the image origin is at - // the bottom-left corner, whereas MediaPipe in general assumes the image origin is at the - // top-left corner. - // NOTE: use "flipFramesVertically" in manifest metadata to override this behavior. - private static final boolean FLIP_FRAMES_VERTICALLY = true; - - // Number of output frames allocated in ExternalTextureConverter. - // NOTE: use "converterNumBuffers" in manifest metadata to override number of buffers. For - // example, when there is a FlowLimiterCalculator in the graph, number of buffers should be at - // least `max_in_flight + max_in_queue + 1` (where max_in_flight and max_in_queue are used in - // FlowLimiterCalculator options). That's because we need buffers for all the frames that are in - // flight/queue plus one for the next frame from the camera. - private static final int NUM_BUFFERS = 2; - - static { - // Load all native libraries needed by the app. - System.loadLibrary("mediapipe_jni"); - try { - System.loadLibrary("opencv_java3"); - } catch (java.lang.UnsatisfiedLinkError e) { - // Some example apps (e.g. template matching) require OpenCV 4. - System.loadLibrary("opencv_java4"); - } - } - - // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed - // frames onto a {@link Surface}. - protected FrameProcessor processor; - // Handles camera access via the {@link CameraX} Jetpack support library. - protected CameraXPreviewHelper cameraHelper; - - // {@link SurfaceTexture} where the camera-preview frames can be accessed. - private SurfaceTexture previewFrameTexture; - // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph. - private SurfaceView previewDisplayView; - - // Creates and manages an {@link EGLContext}. - private EglManager eglManager; - // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be - // consumed by {@link FrameProcessor} and the underlying MediaPipe graph. - private ExternalTextureConverter converter; - - // ApplicationInfo for retrieving metadata defined in the manifest. - private ApplicationInfo applicationInfo; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - setContentView(getContentViewLayoutResId()); - - try { - applicationInfo = - getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA); - } catch (NameNotFoundException e) { - Log.e(TAG, "Cannot find application info: " + e); - } - - previewDisplayView = new SurfaceView(this); - setupPreviewDisplayView(); - - // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g., - // binary graphs. - AndroidAssetUtil.initializeNativeAssetManager(this); - eglManager = new EglManager(null); - processor = - new FrameProcessor( - this, - eglManager.getNativeContext(), - applicationInfo.metaData.getString("binaryGraphName"), - applicationInfo.metaData.getString("inputVideoStreamName"), - applicationInfo.metaData.getString("outputVideoStreamName")); - processor - .getVideoSurfaceOutput() - .setFlipY( - applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY)); - - PermissionHelper.checkAndRequestCameraPermissions(this); - } - - // Used to obtain the content view for this application. If you are extending this class, and - // have a custom layout, override this method and return the custom layout. - protected int getContentViewLayoutResId() { - return R.layout.activity_main; - } - - @Override - protected void onResume() { - super.onResume(); - converter = - new ExternalTextureConverter( - eglManager.getContext(), - applicationInfo.metaData.getInt("converterNumBuffers", NUM_BUFFERS)); - converter.setFlipY( - applicationInfo.metaData.getBoolean("flipFramesVertically", FLIP_FRAMES_VERTICALLY)); - converter.setConsumer(processor); - if (PermissionHelper.cameraPermissionsGranted(this)) { - startCamera(); - } - } - - @Override - protected void onPause() { - super.onPause(); - converter.close(); - - // Hide preview display until we re-open the camera again. - previewDisplayView.setVisibility(View.GONE); - } - - @Override - public void onRequestPermissionsResult( - int requestCode, String[] permissions, int[] grantResults) { - super.onRequestPermissionsResult(requestCode, permissions, grantResults); - PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults); - } - - protected void onCameraStarted(SurfaceTexture surfaceTexture) { - previewFrameTexture = surfaceTexture; - // Make the display view visible to start showing the preview. This triggers the - // SurfaceHolder.Callback added to (the holder of) previewDisplayView. - previewDisplayView.setVisibility(View.VISIBLE); - } - - protected Size cameraTargetResolution() { - return null; // No preference and let the camera (helper) decide. - } - - public void startCamera() { - cameraHelper = new CameraXPreviewHelper(); - cameraHelper.setOnCameraStartedListener( - surfaceTexture -> { - onCameraStarted(surfaceTexture); - }); - CameraHelper.CameraFacing cameraFacing = - applicationInfo.metaData.getBoolean("cameraFacingFront", false) - ? CameraHelper.CameraFacing.FRONT - : CameraHelper.CameraFacing.BACK; - cameraHelper.startCamera( - this, cameraFacing, /*unusedSurfaceTexture=*/ null, cameraTargetResolution()); - } - - protected Size computeViewSize(int width, int height) { - return new Size(width, height); - } - - protected void onPreviewDisplaySurfaceChanged( - SurfaceHolder holder, int format, int width, int height) { - // (Re-)Compute the ideal size of the camera-preview display (the area that the - // camera-preview frames get rendered onto, potentially with scaling and rotation) - // based on the size of the SurfaceView that contains the display. - Size viewSize = computeViewSize(width, height); - Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize); - boolean isCameraRotated = cameraHelper.isCameraRotated(); - - // Connect the converter to the camera-preview frames as its input (via - // previewFrameTexture), and configure the output width and height as the computed - // display size. - converter.setSurfaceTextureAndAttachToGLContext( - previewFrameTexture, - isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(), - isCameraRotated ? displaySize.getWidth() : displaySize.getHeight()); - } - - private void setupPreviewDisplayView() { - previewDisplayView.setVisibility(View.GONE); - ViewGroup viewGroup = findViewById(R.id.preview_display_layout); - viewGroup.addView(previewDisplayView); - - previewDisplayView - .getHolder() - .addCallback( - new SurfaceHolder.Callback() { - @Override - public void surfaceCreated(SurfaceHolder holder) { - processor.getVideoSurfaceOutput().setSurface(holder.getSurface()); - } - - @Override - public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { - onPreviewDisplaySurfaceChanged(holder, format, width, height); - } - - @Override - public void surfaceDestroyed(SurfaceHolder holder) { - processor.getVideoSurfaceOutput().setSurface(null); - } - }); - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable-v24/ic_launcher_foreground.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable-v24/ic_launcher_foreground.xml deleted file mode 100644 index c7bd21dbd..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable-v24/ic_launcher_foreground.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable/ic_launcher_background.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable/ic_launcher_background.xml deleted file mode 100644 index 01f0af0ad..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/drawable/ic_launcher_background.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/layout/activity_main.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/layout/activity_main.xml deleted file mode 100644 index c19d7e628..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/layout/activity_main.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher.xml deleted file mode 100644 index d372a4fca..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher_round.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher_round.xml deleted file mode 100644 index d372a4fca..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-anydpi-v26/ic_launcher_round.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher.png deleted file mode 100644 index 920cbf3f1..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_foreground.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_foreground.png deleted file mode 100644 index 7dfa35f78..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_round.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_round.png deleted file mode 100644 index 20dfc6c13..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-hdpi/ic_launcher_round.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher.png deleted file mode 100644 index 3813caa21..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_foreground.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_foreground.png deleted file mode 100644 index d49232778..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_round.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_round.png deleted file mode 100644 index 18f9ac5e7..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-mdpi/ic_launcher_round.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher.png deleted file mode 100644 index b79cc77a0..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_foreground.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_foreground.png deleted file mode 100644 index 811b17cee..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_round.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_round.png deleted file mode 100644 index 2f3c752e5..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xhdpi/ic_launcher_round.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher.png deleted file mode 100644 index 05585e4c6..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_foreground.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_foreground.png deleted file mode 100644 index 46b77639c..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_round.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_round.png deleted file mode 100644 index f87332424..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxhdpi/ic_launcher_round.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher.png deleted file mode 100644 index 8ccd6c840..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_foreground.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_foreground.png deleted file mode 100644 index 06b982e31..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_round.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_round.png deleted file mode 100644 index 501748acc..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/mipmap-xxxhdpi/ic_launcher_round.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/colors.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/colors.xml deleted file mode 100644 index 69b22338c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/colors.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - #008577 - #00574B - #D81B60 - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/strings.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/strings.xml deleted file mode 100644 index e45fdb48f..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/strings.xml +++ /dev/null @@ -1,3 +0,0 @@ - - Please grant camera permissions. - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/styles.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/styles.xml deleted file mode 100644 index 5885930df..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/res/values/styles.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD deleted file mode 100644 index 279d29b74..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/face_detection:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "facedetectioncpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/face_detection:face_detection_mobile_cpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.facedetectioncpu", - "appName": "Face Detection (CPU)", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "face_detection_mobile_cpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD deleted file mode 100644 index 11351fc56..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/face_detection:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "facedetectiongpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/face_detection:face_detection_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.facedetectiongpu", - "appName": "Face Detection", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "face_detection_mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD deleted file mode 100644 index 8bf6c0a54..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/BUILD +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/face_effect:face_effect_gpu_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "faceeffect", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/face_effect/data:axis.binarypb", - "//mediapipe/graphs/face_effect/data:axis.pngblob", - "//mediapipe/graphs/face_effect/data:facepaint.pngblob", - "//mediapipe/graphs/face_effect/data:glasses.binarypb", - "//mediapipe/graphs/face_effect/data:glasses.pngblob", - "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.faceeffect", - "appName": "Face Effect", - "mainActivity": ".MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "face_effect_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:matrix_data_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java deleted file mode 100644 index 78c220aae..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/faceeffect/MainActivity.java +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.faceeffect; - -import android.graphics.Color; -import android.os.Bundle; -import android.util.Log; -import android.view.GestureDetector; -import android.view.Gravity; -import android.view.MotionEvent; -import android.view.View; -import android.view.ViewGroup; -import android.view.ViewGroup.LayoutParams; -import android.widget.RelativeLayout; -import android.widget.TextView; -import com.google.mediapipe.framework.Packet; -import com.google.mediapipe.framework.PacketGetter; -import com.google.mediapipe.modules.facegeometry.FaceGeometryProto.FaceGeometry; -import com.google.mediapipe.formats.proto.MatrixDataProto.MatrixData; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Main activity of MediaPipe face mesh app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - // Side packet / stream names. - private static final String USE_FACE_DETECTION_INPUT_SOURCE_INPUT_SIDE_PACKET_NAME = - "use_face_detection_input_source"; - private static final String SELECTED_EFFECT_ID_INPUT_STREAM_NAME = "selected_effect_id"; - private static final String OUTPUT_FACE_GEOMETRY_STREAM_NAME = "multi_face_geometry"; - - private static final String EFFECT_SWITCHING_HINT_TEXT = "Tap to switch between effects!"; - - private static final boolean USE_FACE_DETECTION_INPUT_SOURCE = false; - private static final int MATRIX_TRANSLATION_Z_INDEX = 14; - - private static final int SELECTED_EFFECT_ID_AXIS = 0; - private static final int SELECTED_EFFECT_ID_FACEPAINT = 1; - private static final int SELECTED_EFFECT_ID_GLASSES = 2; - - private final Object effectSelectionLock = new Object(); - private int selectedEffectId; - - private View effectSwitchingHintView; - private GestureDetector tapGestureDetector; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - // Add an effect switching hint view to the preview layout. - effectSwitchingHintView = createEffectSwitchingHintView(); - effectSwitchingHintView.setVisibility(View.INVISIBLE); - ViewGroup viewGroup = findViewById(R.id.preview_display_layout); - viewGroup.addView(effectSwitchingHintView); - - // By default, render the axis effect for the face detection input source and the glasses effect - // for the face landmark input source. - if (USE_FACE_DETECTION_INPUT_SOURCE) { - selectedEffectId = SELECTED_EFFECT_ID_AXIS; - } else { - selectedEffectId = SELECTED_EFFECT_ID_GLASSES; - } - - // Pass the USE_FACE_DETECTION_INPUT_SOURCE flag value as an input side packet into the graph. - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put( - USE_FACE_DETECTION_INPUT_SOURCE_INPUT_SIDE_PACKET_NAME, - processor.getPacketCreator().createBool(USE_FACE_DETECTION_INPUT_SOURCE)); - processor.setInputSidePackets(inputSidePackets); - - // This callback demonstrates how the output face geometry packet can be obtained and used - // in an Android app. As an example, the Z-translation component of the face pose transform - // matrix is logged for each face being equal to the approximate distance away from the camera - // in centimeters. - processor.addPacketCallback( - OUTPUT_FACE_GEOMETRY_STREAM_NAME, - (packet) -> { - effectSwitchingHintView.post( - () -> - effectSwitchingHintView.setVisibility( - USE_FACE_DETECTION_INPUT_SOURCE ? View.INVISIBLE : View.VISIBLE)); - - Log.d(TAG, "Received a multi face geometry packet."); - List multiFaceGeometry = - PacketGetter.getProtoVector(packet, FaceGeometry.parser()); - - StringBuilder approxDistanceAwayFromCameraLogMessage = new StringBuilder(); - for (FaceGeometry faceGeometry : multiFaceGeometry) { - if (approxDistanceAwayFromCameraLogMessage.length() > 0) { - approxDistanceAwayFromCameraLogMessage.append(' '); - } - MatrixData poseTransformMatrix = faceGeometry.getPoseTransformMatrix(); - approxDistanceAwayFromCameraLogMessage.append( - -poseTransformMatrix.getPackedData(MATRIX_TRANSLATION_Z_INDEX)); - } - - Log.d( - TAG, - "[TS:" - + packet.getTimestamp() - + "] size = " - + multiFaceGeometry.size() - + "; approx. distance away from camera in cm for faces = [" - + approxDistanceAwayFromCameraLogMessage - + "]"); - }); - - // Alongside the input camera frame, we also send the `selected_effect_id` int32 packet to - // indicate which effect should be rendered on this frame. - processor.setOnWillAddFrameListener( - (timestamp) -> { - Packet selectedEffectIdPacket = null; - try { - synchronized (effectSelectionLock) { - selectedEffectIdPacket = processor.getPacketCreator().createInt32(selectedEffectId); - } - - processor - .getGraph() - .addPacketToInputStream( - SELECTED_EFFECT_ID_INPUT_STREAM_NAME, selectedEffectIdPacket, timestamp); - } catch (RuntimeException e) { - Log.e( - TAG, "Exception while adding packet to input stream while switching effects: " + e); - } finally { - if (selectedEffectIdPacket != null) { - selectedEffectIdPacket.release(); - } - } - }); - - // We use the tap gesture detector to switch between face effects. This allows users to try - // multiple pre-bundled face effects without a need to recompile the app. - tapGestureDetector = - new GestureDetector( - this, - new GestureDetector.SimpleOnGestureListener() { - @Override - public void onLongPress(MotionEvent event) { - switchEffect(); - } - - @Override - public boolean onSingleTapUp(MotionEvent event) { - switchEffect(); - return true; - } - - private void switchEffect() { - // Avoid switching the Axis effect for the face detection input source. - if (USE_FACE_DETECTION_INPUT_SOURCE) { - return; - } - - // Looped effect order: glasses -> facepaint -> axis -> glasses -> ... - synchronized (effectSelectionLock) { - switch (selectedEffectId) { - case SELECTED_EFFECT_ID_AXIS: - { - selectedEffectId = SELECTED_EFFECT_ID_GLASSES; - break; - } - - case SELECTED_EFFECT_ID_FACEPAINT: - { - selectedEffectId = SELECTED_EFFECT_ID_AXIS; - break; - } - - case SELECTED_EFFECT_ID_GLASSES: - { - selectedEffectId = SELECTED_EFFECT_ID_FACEPAINT; - break; - } - - default: - break; - } - } - } - }); - } - - @Override - public boolean onTouchEvent(MotionEvent event) { - return tapGestureDetector.onTouchEvent(event); - } - - private View createEffectSwitchingHintView() { - TextView effectSwitchingHintView = new TextView(getApplicationContext()); - effectSwitchingHintView.setLayoutParams( - new RelativeLayout.LayoutParams(LayoutParams.FILL_PARENT, LayoutParams.FILL_PARENT)); - effectSwitchingHintView.setText(EFFECT_SWITCHING_HINT_TEXT); - effectSwitchingHintView.setGravity(Gravity.CENTER_HORIZONTAL | Gravity.BOTTOM); - effectSwitchingHintView.setPadding(0, 0, 0, 480); - effectSwitchingHintView.setTextColor(Color.parseColor("#ffffff")); - effectSwitchingHintView.setTextSize((float) 24); - - return effectSwitchingHintView; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD deleted file mode 100644 index 26406e77b..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/BUILD +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/face_mesh:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "facemeshgpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.facemeshgpu", - "appName": "Face Mesh", - "mainActivity": ".MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "face_mesh_mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java deleted file mode 100644 index 82c1f4478..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facemeshgpu/MainActivity.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.facemeshgpu; - -import android.os.Bundle; -import android.util.Log; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; -import com.google.mediapipe.framework.AndroidPacketCreator; -import com.google.mediapipe.framework.Packet; -import com.google.mediapipe.framework.PacketGetter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Main activity of MediaPipe face mesh app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - private static final String INPUT_NUM_FACES_SIDE_PACKET_NAME = "num_faces"; - private static final String OUTPUT_LANDMARKS_STREAM_NAME = "multi_face_landmarks"; - // Max number of faces to detect/process. - private static final int NUM_FACES = 1; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - AndroidPacketCreator packetCreator = processor.getPacketCreator(); - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put(INPUT_NUM_FACES_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_FACES)); - processor.setInputSidePackets(inputSidePackets); - - // To show verbose logging, run: - // adb shell setprop log.tag.MainActivity VERBOSE - if (Log.isLoggable(TAG, Log.VERBOSE)) { - processor.addPacketCallback( - OUTPUT_LANDMARKS_STREAM_NAME, - (packet) -> { - Log.v(TAG, "Received multi face landmarks packet."); - List multiFaceLandmarks = - PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser()); - Log.v( - TAG, - "[TS:" - + packet.getTimestamp() - + "] " - + getMultiFaceLandmarksDebugString(multiFaceLandmarks)); - }); - } - } - - private static String getMultiFaceLandmarksDebugString( - List multiFaceLandmarks) { - if (multiFaceLandmarks.isEmpty()) { - return "No face landmarks"; - } - String multiFaceLandmarksStr = "Number of faces detected: " + multiFaceLandmarks.size() + "\n"; - int faceIndex = 0; - for (NormalizedLandmarkList landmarks : multiFaceLandmarks) { - multiFaceLandmarksStr += - "\t#Face landmarks for face[" + faceIndex + "]: " + landmarks.getLandmarkCount() + "\n"; - int landmarkIndex = 0; - for (NormalizedLandmark landmark : landmarks.getLandmarkList()) { - multiFaceLandmarksStr += - "\t\tLandmark [" - + landmarkIndex - + "]: (" - + landmark.getX() - + ", " - + landmark.getY() - + ", " - + landmark.getZ() - + ")\n"; - ++landmarkIndex; - } - ++faceIndex; - } - return multiFaceLandmarksStr; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD deleted file mode 100644 index df58f2713..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/hair_segmentation:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "hairsegmentationgpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/hair_segmentation:mobile_gpu.binarypb", - "//mediapipe/models:hair_segmentation.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.hairsegmentationgpu", - "appName": "Hair Segmentation", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD deleted file mode 100644 index 2d9813301..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/hand_tracking:detection_mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "handdetectiongpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/hand_tracking:hand_detection_mobile_gpu.binarypb", - "//mediapipe/modules/palm_detection:palm_detection.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.handdetectiongpu", - "appName": "Hand Detection", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "hand_detection_mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD deleted file mode 100644 index 7b3bfe847..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/hand_tracking:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "handtrackinggpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu.binarypb", - "//mediapipe/modules/hand_landmark:handedness.txt", - "//mediapipe/modules/hand_landmark:hand_landmark.tflite", - "//mediapipe/modules/palm_detection:palm_detection.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.handtrackinggpu", - "appName": "Hand Tracking", - "mainActivity": ".MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "hand_tracking_mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java deleted file mode 100644 index 445431bc4..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.handtrackinggpu; - -import android.os.Bundle; -import android.util.Log; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; -import com.google.mediapipe.framework.AndroidPacketCreator; -import com.google.mediapipe.framework.Packet; -import com.google.mediapipe.framework.PacketGetter; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Main activity of MediaPipe hand tracking app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - private static final String INPUT_NUM_HANDS_SIDE_PACKET_NAME = "num_hands"; - private static final String OUTPUT_LANDMARKS_STREAM_NAME = "hand_landmarks"; - // Max number of hands to detect/process. - private static final int NUM_HANDS = 2; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - AndroidPacketCreator packetCreator = processor.getPacketCreator(); - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put(INPUT_NUM_HANDS_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_HANDS)); - processor.setInputSidePackets(inputSidePackets); - - // To show verbose logging, run: - // adb shell setprop log.tag.MainActivity VERBOSE - if (Log.isLoggable(TAG, Log.VERBOSE)) { - processor.addPacketCallback( - OUTPUT_LANDMARKS_STREAM_NAME, - (packet) -> { - Log.v(TAG, "Received multi-hand landmarks packet."); - List multiHandLandmarks = - PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser()); - Log.v( - TAG, - "[TS:" - + packet.getTimestamp() - + "] " - + getMultiHandLandmarksDebugString(multiHandLandmarks)); - }); - } - } - - private String getMultiHandLandmarksDebugString(List multiHandLandmarks) { - if (multiHandLandmarks.isEmpty()) { - return "No hand landmarks"; - } - String multiHandLandmarksStr = "Number of hands detected: " + multiHandLandmarks.size() + "\n"; - int handIndex = 0; - for (NormalizedLandmarkList landmarks : multiHandLandmarks) { - multiHandLandmarksStr += - "\t#Hand landmarks for hand[" + handIndex + "]: " + landmarks.getLandmarkCount() + "\n"; - int landmarkIndex = 0; - for (NormalizedLandmark landmark : landmarks.getLandmarkList()) { - multiHandLandmarksStr += - "\t\tLandmark [" - + landmarkIndex - + "]: (" - + landmark.getX() - + ", " - + landmark.getY() - + ", " - + landmark.getZ() - + ")\n"; - ++landmarkIndex; - } - ++handIndex; - } - return multiHandLandmarksStr; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD deleted file mode 100644 index d9c660088..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/holistictrackinggpu/BUILD +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "holistictrackinggpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/hand_landmark:hand_landmark.tflite", - "//mediapipe/modules/hand_landmark:handedness.txt", - "//mediapipe/modules/holistic_landmark:hand_recrop.tflite", - "//mediapipe/modules/pose_detection:pose_detection.tflite", - "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.holistictrackinggpu", - "appName": "Holistic Tracking", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "holistic_tracking_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "3", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD deleted file mode 100644 index 3dea64053..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/BUILD +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -java_lite_proto_library( - name = "sticker_buffer_java_proto_lite", - deps = ["//mediapipe/graphs/instant_motion_tracking/calculators:sticker_buffer_proto"], -) - -android_library( - name = "instantmotiontracking_lib", - srcs = glob(["*.java"]), - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - resource_files = glob([ - "res/layout/**", - "res/drawable/**", - ]), - visibility = ["//visibility:public"], - deps = [ - ":sticker_buffer_java_proto_lite", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/java/com/google/mediapipe/components:android_components", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "//third_party:androidx_appcompat", - "//third_party:androidx_core", - "//third_party:opencv", - "@maven//:androidx_concurrent_concurrent_futures", - "@maven//:com_github_bumptech_glide_glide", - "@maven//:com_google_guava_guava", - ], -) - -# Include all calculators specific to this project defined by BUILD in graphs -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/instant_motion_tracking:instant_motion_tracking_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -# Converts the .so cc_binary into a cc_library, to be consumed in an android_binary. -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -genrule( - name = "asset3d", - srcs = ["//mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets:robot/robot.obj.uuu.zip"], - outs = ["robot/robot.obj.uuu"], - cmd = "unzip -p $< > $@", -) - -android_binary( - name = "instantmotiontracking", - assets = [ - ":asset3d", - "//mediapipe/graphs/instant_motion_tracking:instant_motion_tracking.binarypb", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets:gif/gif.obj.uuu", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets:gif/default_gif_texture.jpg", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets:robot/robot_texture.jpg", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.instantmotiontracking", - "appName": "Instant Motion Tracking", - "mainActivity": ".MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "instant_motion_tracking.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":instantmotiontracking_lib", - ":mediapipe_jni_lib", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/GIFEditText.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/GIFEditText.java deleted file mode 100644 index 1b733ed82..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/GIFEditText.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.instantmotiontracking; - -import android.content.ClipDescription; -import android.content.Context; -import android.net.Uri; -import android.os.Bundle; -import androidx.appcompat.widget.AppCompatEditText; -import android.util.AttributeSet; -import android.util.Log; -import android.view.inputmethod.EditorInfo; -import android.view.inputmethod.InputConnection; -import androidx.core.view.inputmethod.EditorInfoCompat; -import androidx.core.view.inputmethod.InputConnectionCompat; -import androidx.core.view.inputmethod.InputContentInfoCompat; - -// import android.support.v13.view.inputmethod.EditorInfoCompat; -// import android.support.v13.view.inputmethod.InputConnectionCompat; -// import android.support.v13.view.inputmethod.InputContentInfoCompat; - -/** - * This custom EditText implementation uses the existing EditText framework in - * order to develop a GIFEditText input box which is capable of accepting GIF - * animations from the Android system keyboard and return the GIF location with - * a content URI. - */ -public class GIFEditText extends AppCompatEditText { - - private GIFCommitListener gifCommitListener; - - public GIFEditText(Context context) { - super(context); - } - - public GIFEditText(Context context, AttributeSet attrs) { - super(context, attrs); - } - - /** - * onGIFCommit is called once content is pushed to the EditText via the - * Android keyboard. - */ - public interface GIFCommitListener { - void onGIFCommit(Uri contentUri, ClipDescription description); - } - - /** - * Used to set the gifCommitListener for this GIFEditText. - * - * @param gifCommitListener handles response to new content pushed to EditText - */ - public void setGIFCommitListener(GIFCommitListener gifCommitListener) { - this.gifCommitListener = gifCommitListener; - } - - @Override - public InputConnection onCreateInputConnection(EditorInfo editorInfo) { - final InputConnection inputConnection = super.onCreateInputConnection(editorInfo); - EditorInfoCompat.setContentMimeTypes(editorInfo, new String[] {"image/gif"}); - return InputConnectionCompat.createWrapper( - inputConnection, - editorInfo, - new InputConnectionCompat.OnCommitContentListener() { - @Override - public boolean onCommitContent( - final InputContentInfoCompat inputContentInfo, int flags, Bundle opts) { - try { - if (gifCommitListener != null) { - Runnable runnable = - new Runnable() { - @Override - public void run() { - inputContentInfo.requestPermission(); - gifCommitListener.onGIFCommit( - inputContentInfo.getContentUri(), inputContentInfo.getDescription()); - inputContentInfo.releasePermission(); - } - }; - new Thread(runnable).start(); - } - } catch (RuntimeException e) { - Log.e("GIFEditText", "Input connection to GIF selection failed"); - e.printStackTrace(); - return false; - } - return true; - } - }); - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/MainActivity.java deleted file mode 100644 index b4c3f46dc..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/MainActivity.java +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.instantmotiontracking; - -import static java.lang.Math.max; - -import android.content.ClipDescription; -import android.content.Context; -import android.content.Intent; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.graphics.Color; -import android.graphics.Matrix; -import android.graphics.drawable.Drawable; -import android.hardware.Sensor; -import android.hardware.SensorEvent; -import android.hardware.SensorEventListener; -import android.hardware.SensorManager; -import android.net.Uri; -import android.os.Bundle; -import android.util.Log; -import android.util.Size; -import android.view.MotionEvent; -import android.view.SurfaceHolder; -import android.view.View; -import android.view.ViewGroup; -import android.view.inputmethod.InputMethodManager; -import android.widget.ImageButton; -import android.widget.ImageView; -import android.widget.LinearLayout; -import com.bumptech.glide.Glide; -import com.bumptech.glide.load.resource.gif.GifDrawable; -import com.bumptech.glide.request.target.CustomTarget; -import com.bumptech.glide.request.transition.Transition; -import com.google.mediapipe.components.FrameProcessor; -import com.google.mediapipe.framework.AndroidPacketCreator; -import com.google.mediapipe.framework.Packet; -import java.io.InputStream; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * This is the MainActivity that handles camera input, IMU sensor data acquisition - * and sticker management for the InstantMotionTracking MediaPipe project. - */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "InstantMotionTrackingMainActivity"; - - // Allows for automated packet transmission to graph - private MediaPipePacketManager mediaPipePacketManager; - - private static final int TARGET_CAMERA_WIDTH = 960; - private static final int TARGET_CAMERA_HEIGHT = 1280; - private static final float TARGET_CAMERA_ASPECT_RATIO = - (float) TARGET_CAMERA_WIDTH / (float) TARGET_CAMERA_HEIGHT; - - // Bounds for a single click (sticker anchor reset) - private static final long CLICK_DURATION = 300; // ms - private long clickStartMillis = 0; - private ViewGroup viewGroup; - // Contains dynamic layout of sticker data controller - private LinearLayout buttonLayout; - - private ArrayList stickerArrayList; - // Current sticker being edited by user - private StickerManager currentSticker; - // Trip value used to determine sticker re-anchoring - private static final String STICKER_SENTINEL_TAG = "sticker_sentinel"; - private int stickerSentinel = -1; - - // Define parameters for 'reactivity' of object - private static final float ROTATION_SPEED = 5.0f; - private static final float SCALING_FACTOR = 0.025f; - - // Parameters of device visual field for rendering system - // (68 degrees, 4:3 for Pixel 4) - // TODO : Make acquisition of this information automated - private static final float VERTICAL_FOV_RADIANS = (float) Math.toRadians(68.0); - private static final String FOV_SIDE_PACKET_TAG = "vertical_fov_radians"; - private static final String ASPECT_RATIO_SIDE_PACKET_TAG = "aspect_ratio"; - - private static final String IMU_MATRIX_TAG = "imu_rotation_matrix"; - private static final int SENSOR_SAMPLE_DELAY = SensorManager.SENSOR_DELAY_FASTEST; - private final float[] rotationMatrix = new float[9]; - - private static final String STICKER_PROTO_TAG = "sticker_proto_string"; - // Assets for object rendering - // All animation assets and tags for the first asset (1) - private Bitmap asset3dTexture = null; - private static final String ASSET_3D_TEXTURE = "robot/robot_texture.jpg"; - private static final String ASSET_3D_FILE = "robot/robot.obj.uuu"; - private static final String ASSET_3D_TEXTURE_TAG = "texture_3d"; - private static final String ASSET_3D_TAG = "asset_3d"; - // All GIF animation assets and tags - private GIFEditText editText; - private ArrayList gifBitmaps = new ArrayList<>(); - private int gifCurrentIndex = 0; - private Bitmap defaultGIFTexture = null; // Texture sent if no gif available - // last time the GIF was updated - private long gifLastFrameUpdateMS = System.currentTimeMillis(); - private static final int GIF_FRAME_RATE = 20; // 20 FPS - private static final String GIF_ASPECT_RATIO_TAG = "gif_aspect_ratio"; - private static final String DEFAULT_GIF_TEXTURE = "gif/default_gif_texture.jpg"; - private static final String GIF_FILE = "gif/gif.obj.uuu"; - private static final String GIF_TEXTURE_TAG = "gif_texture"; - private static final String GIF_ASSET_TAG = "gif_asset_name"; - - private int cameraWidth = TARGET_CAMERA_WIDTH; - private int cameraHeight = TARGET_CAMERA_HEIGHT; - - @Override - protected Size cameraTargetResolution() { - // Camera size is in landscape, so here we have (height, width) - return new Size(TARGET_CAMERA_HEIGHT, TARGET_CAMERA_WIDTH); - } - - @Override - protected Size computeViewSize(int width, int height) { - // Try to force aspect ratio of view size to match our target aspect ratio - return new Size(height, (int) (height * TARGET_CAMERA_ASPECT_RATIO)); - } - - @Override - protected void onPreviewDisplaySurfaceChanged( - SurfaceHolder holder, int format, int width, int height) { - super.onPreviewDisplaySurfaceChanged(holder, format, width, height); - boolean isCameraRotated = cameraHelper.isCameraRotated(); - - // cameraImageSize computation logic duplicated from base MainActivity - Size viewSize = computeViewSize(width, height); - Size cameraImageSize = cameraHelper.computeDisplaySizeFromViewSize(viewSize); - cameraWidth = - isCameraRotated ? cameraImageSize.getHeight() : cameraImageSize.getWidth(); - cameraHeight = - isCameraRotated ? cameraImageSize.getWidth() : cameraImageSize.getHeight(); - } - - @Override - protected void onCreate(Bundle savedInstanceState) { - - super.onCreate(savedInstanceState); - - editText = findViewById(R.id.gif_edit_text); - editText.setGIFCommitListener( - new GIFEditText.GIFCommitListener() { - @Override - public void onGIFCommit(Uri contentUri, ClipDescription description) { - // The application must have permission to access the GIF content - grantUriPermission( - "com.google.mediapipe.apps.instantmotiontracking", - contentUri, - Intent.FLAG_GRANT_READ_URI_PERMISSION); - // Set GIF frames from content URI - setGIFBitmaps(contentUri.toString()); - // Close the keyboard upon GIF acquisition - closeKeyboard(); - } - }); - - // Send loaded 3d render assets as side packets to graph - prepareDemoAssets(); - AndroidPacketCreator packetCreator = processor.getPacketCreator(); - - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put(ASSET_3D_TEXTURE_TAG, - packetCreator.createRgbaImageFrame(asset3dTexture)); - inputSidePackets.put(ASSET_3D_TAG, - packetCreator.createString(ASSET_3D_FILE)); - inputSidePackets.put(GIF_ASSET_TAG, - packetCreator.createString(GIF_FILE)); - processor.setInputSidePackets(inputSidePackets); - - // Add frame listener to PacketManagement system - mediaPipePacketManager = new MediaPipePacketManager(); - processor.setOnWillAddFrameListener(mediaPipePacketManager); - - // Send device properties to render objects via OpenGL - Map devicePropertiesSidePackets = new HashMap<>(); - // TODO: Note that if our actual camera stream resolution does not match the - // requested aspect ratio, then we will need to update the value used for - // this packet, or else tracking results will be off. - devicePropertiesSidePackets.put( - ASPECT_RATIO_SIDE_PACKET_TAG, packetCreator.createFloat32(TARGET_CAMERA_ASPECT_RATIO)); - devicePropertiesSidePackets.put( - FOV_SIDE_PACKET_TAG, packetCreator.createFloat32(VERTICAL_FOV_RADIANS)); - processor.setInputSidePackets(devicePropertiesSidePackets); - - // Begin with 0 stickers in dataset - stickerArrayList = new ArrayList<>(); - currentSticker = null; - - SensorManager sensorManager = (SensorManager) getSystemService(SENSOR_SERVICE); - List sensorList = sensorManager.getSensorList(Sensor.TYPE_ROTATION_VECTOR); - sensorManager.registerListener( - new SensorEventListener() { - private final float[] rotMatFromVec = new float[9]; - - @Override - public void onAccuracyChanged(Sensor sensor, int accuracy) {} - // Update procedure on sensor adjustment (phone changes orientation) - - @Override - public void onSensorChanged(SensorEvent event) { - // Get the Rotation Matrix from the Rotation Vector - SensorManager.getRotationMatrixFromVector(rotMatFromVec, event.values); - // AXIS_MINUS_X is used to remap the rotation matrix for left hand - // rules in the MediaPipe graph - SensorManager.remapCoordinateSystem( - rotMatFromVec, SensorManager.AXIS_MINUS_X, SensorManager.AXIS_Y, rotationMatrix); - } - }, - (Sensor) sensorList.get(0), - SENSOR_SAMPLE_DELAY); - - // Mechanisms for zoom, pinch, rotation, tap gestures - buttonLayout = (LinearLayout) findViewById(R.id.button_layout); - viewGroup = findViewById(R.id.preview_display_layout); - viewGroup.setOnTouchListener( - new View.OnTouchListener() { - @Override - public boolean onTouch(View v, MotionEvent event) { - return manageUiTouch(event); - } - }); - refreshUi(); - } - - // Obtain our custom activity_main layout for InstantMotionTracking - @Override - protected int getContentViewLayoutResId() { - return R.layout.instant_motion_tracking_activity_main; - } - - // Manages a touch event in order to perform placement/rotation/scaling gestures - // on virtual sticker objects. - private boolean manageUiTouch(MotionEvent event) { - if (currentSticker != null) { - switch (event.getAction()) { - // Detecting a single click for object re-anchoring - case (MotionEvent.ACTION_DOWN): - clickStartMillis = System.currentTimeMillis(); - break; - case (MotionEvent.ACTION_UP): - if (System.currentTimeMillis() - clickStartMillis <= CLICK_DURATION) { - recordClick(event); - } - break; - case (MotionEvent.ACTION_MOVE): - // Rotation and Scaling are independent events and can occur simulataneously - if (event.getPointerCount() == 2) { - if (event.getHistorySize() > 1) { - // Calculate user scaling of sticker - float newScaleFactor = getNewScaleFactor(event, currentSticker.getScaleFactor()); - currentSticker.setScaleFactor(newScaleFactor); - // calculate rotation (radians) for dynamic y-axis rotations - float rotationIncrement = calculateRotationRadians(event); - currentSticker.setRotation(currentSticker.getRotation() + rotationIncrement); - } - } - break; - default: - // fall out - } - } - return true; - } - - // Returns a float value that is equal to the radians of rotation from a two-finger - // MotionEvent recorded by the OnTouchListener. - private static float calculateRotationRadians(MotionEvent event) { - float tangentA = - (float) Math.atan2(event.getY(1) - event.getY(0), event.getX(1) - event.getX(0)); - float tangentB = - (float) - Math.atan2( - event.getHistoricalY(1, 0) - event.getHistoricalY(0, 0), - event.getHistoricalX(1, 0) - event.getHistoricalX(0, 0)); - float angle = ((float) Math.toDegrees(tangentA - tangentB)) % 360f; - angle += ((angle < -180f) ? +360f : ((angle > 180f) ? -360f : 0.0f)); - float rotationIncrement = (float) (Math.PI * ((angle * ROTATION_SPEED) / 180)); - return rotationIncrement; - } - - // Returns a float value that is equal to the translation distance between - // two-fingers that move in a pinch/spreading direction. - private static float getNewScaleFactor(MotionEvent event, float currentScaleFactor) { - double newDistance = getDistance(event.getX(0), event.getY(0), event.getX(1), event.getY(1)); - double oldDistance = - getDistance( - event.getHistoricalX(0, 0), - event.getHistoricalY(0, 0), - event.getHistoricalX(1, 0), - event.getHistoricalY(1, 0)); - float signFloat = - (newDistance < oldDistance) - ? -SCALING_FACTOR - : SCALING_FACTOR; // Are they moving towards each other? - currentScaleFactor *= (1f + signFloat); - return currentScaleFactor; - } - - // Called if a single touch event is recorded on the screen and used to set the - // new anchor position for the current sticker in focus. - private void recordClick(MotionEvent event) { - // First normalize our click position w.r.t. to the view display - float x = (event.getX() / viewGroup.getWidth()); - float y = (event.getY() / viewGroup.getHeight()); - - // MediaPipe can automatically crop our camera stream when displaying it to - // our surface, which can throw off our touch point calulations. So we need - // to replicate that logic here. See FrameScaleMode::kFillAndCrop usage in - // gl_quad_renderer.cc for more details. - float widthRatio = (float) viewGroup.getWidth() / (float) cameraWidth; - float heightRatio = (float) viewGroup.getHeight() / (float) cameraHeight; - - float maxRatio = max(widthRatio, heightRatio); - widthRatio /= maxRatio; - heightRatio /= maxRatio; - - // Now we scale by the scale factors, and then reposition (since cropping - // is always centered) - x *= widthRatio; - x += 0.5f * (1.0f - widthRatio); - y *= heightRatio; - y += 0.5f * (1.0f - heightRatio); - - // Finally, we can pass our adjusted x and y points to the StickerManager - currentSticker.setAnchorCoordinate(x, y); - stickerSentinel = currentSticker.getstickerId(); - } - - // Provided the X and Y coordinates of two points, the distance between them - // will be returned. - private static double getDistance(double x1, double y1, double x2, double y2) { - return Math.hypot((y2 - y1), (x2 - x1)); - } - - // Called upon each button click, and used to populate the buttonLayout with the - // current sticker data in addition to sticker controls (delete, remove, back). - private void refreshUi() { - if (currentSticker != null) { // No sticker in view - buttonLayout.removeAllViews(); - ImageButton deleteSticker = new ImageButton(this); - setControlButtonDesign(deleteSticker, R.drawable.baseline_clear_24); - deleteSticker.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - if (currentSticker != null) { - stickerArrayList.remove(currentSticker); - currentSticker = null; - refreshUi(); - } - } - }); - // Go to home sticker menu - ImageButton goBack = new ImageButton(this); - setControlButtonDesign(goBack, R.drawable.baseline_arrow_back_24); - goBack.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - currentSticker = null; - refreshUi(); - } - }); - // Change sticker to next possible render - ImageButton loopRender = new ImageButton(this); - setControlButtonDesign(loopRender, R.drawable.baseline_loop_24); - loopRender.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - currentSticker.setRender(currentSticker.getRender().iterate()); - refreshUi(); - } - }); - buttonLayout.addView(deleteSticker); - buttonLayout.addView(goBack); - buttonLayout.addView(loopRender); - - // Add the GIF search option if current sticker is GIF - if (currentSticker.getRender() == StickerManager.Render.GIF) { - ImageButton gifSearch = new ImageButton(this); - setControlButtonDesign(gifSearch, R.drawable.baseline_search_24); - gifSearch.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - // Clear the text field to prevent text artifacts in GIF selection - editText.setText(""); - // Open the Keyboard to allow user input - openKeyboard(); - } - }); - buttonLayout.addView(gifSearch); - } - } else { - buttonLayout.removeAllViews(); - // Display stickers - for (final StickerManager sticker : stickerArrayList) { - final ImageButton stickerButton = new ImageButton(this); - stickerButton.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - currentSticker = sticker; - refreshUi(); - } - }); - if (sticker.getRender() == StickerManager.Render.GIF) { - setControlButtonDesign(stickerButton, R.drawable.asset_gif_preview); - } else if (sticker.getRender() == StickerManager.Render.ASSET_3D) { - setStickerButtonDesign(stickerButton, R.drawable.asset_3d_preview); - } - - buttonLayout.addView(stickerButton); - } - ImageButton addSticker = new ImageButton(this); - setControlButtonDesign(addSticker, R.drawable.baseline_add_24); - addSticker.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - StickerManager newSticker = new StickerManager(); - stickerArrayList.add(newSticker); - currentSticker = newSticker; - refreshUi(); - } - }); - ImageButton clearStickers = new ImageButton(this); - setControlButtonDesign(clearStickers, R.drawable.baseline_clear_all_24); - clearStickers.setOnClickListener( - new View.OnClickListener() { - @Override - public void onClick(View v) { - stickerArrayList.clear(); - refreshUi(); - } - }); - - buttonLayout.addView(addSticker); - buttonLayout.addView(clearStickers); - } - } - - // Sets ImageButton UI for Control Buttons. - private void setControlButtonDesign(ImageButton btn, int imageDrawable) { - // btn.setImageDrawable(getResources().getDrawable(imageDrawable)); - btn.setImageDrawable(getDrawable(imageDrawable)); - btn.setBackgroundColor(Color.parseColor("#00ffffff")); - btn.setColorFilter(Color.parseColor("#0494a4")); - btn.setLayoutParams(new LinearLayout.LayoutParams(200, 200)); - btn.setPadding(25, 25, 25, 25); - btn.setScaleType(ImageView.ScaleType.FIT_XY); - } - - // Sets ImageButton UI for Sticker Buttons. - private void setStickerButtonDesign(ImageButton btn, int imageDrawable) { - btn.setImageDrawable(getDrawable(imageDrawable)); - btn.setBackground(getDrawable(R.drawable.circle_button)); - btn.setLayoutParams(new LinearLayout.LayoutParams(250, 250)); - btn.setPadding(25, 25, 25, 25); - btn.setScaleType(ImageView.ScaleType.CENTER_INSIDE); - } - - // Used to set ArrayList of Bitmap frames - private void setGIFBitmaps(String gifUrl) { - gifBitmaps = new ArrayList<>(); // Empty the bitmap array - Glide.with(this) - .asGif() - .load(gifUrl) - .into( - new CustomTarget() { - @Override - public void onLoadCleared(Drawable placeholder) {} - - @Override - public void onResourceReady( - GifDrawable resource, Transition transition) { - try { - Object startConstant = resource.getConstantState(); - Field frameManager = startConstant.getClass().getDeclaredField("frameLoader"); - frameManager.setAccessible(true); - Object frameLoader = frameManager.get(startConstant); - Field decoder = frameLoader.getClass().getDeclaredField("gifDecoder"); - decoder.setAccessible(true); - - Object frameObject = (decoder.get(frameLoader)); - for (int i = 0; i < resource.getFrameCount(); i++) { - frameObject.getClass().getMethod("advance").invoke(frameObject); - Bitmap bmp = - (Bitmap) - frameObject.getClass().getMethod("getNextFrame").invoke(frameObject); - gifBitmaps.add(flipHorizontal(bmp)); - } - } catch (Exception e) { - Log.e(TAG, "", e); - } - } - }); - } - - // Bitmaps must be flipped due to native acquisition of frames from Android OS - private static Bitmap flipHorizontal(Bitmap bmp) { - Matrix matrix = new Matrix(); - // Flip Bitmap frames horizontally - matrix.preScale(-1.0f, 1.0f); - return Bitmap.createBitmap(bmp, 0, 0, bmp.getWidth(), bmp.getHeight(), matrix, true); - } - - // Function that is continuously called in order to time GIF frame updates - private void updateGIFFrame() { - long millisPerFrame = 1000 / GIF_FRAME_RATE; - if (System.currentTimeMillis() - gifLastFrameUpdateMS >= millisPerFrame) { - // Update GIF timestamp - gifLastFrameUpdateMS = System.currentTimeMillis(); - // Cycle through every possible frame and avoid a divide by 0 - gifCurrentIndex = gifBitmaps.isEmpty() ? 1 : (gifCurrentIndex + 1) % gifBitmaps.size(); - } - } - - // Called once to popup the Keyboard via Android OS with focus set to editText - private void openKeyboard() { - editText.requestFocus(); - InputMethodManager imm = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE); - imm.showSoftInput(editText, InputMethodManager.SHOW_IMPLICIT); - } - - // Called once to close the Keyboard via Android OS - private void closeKeyboard() { - View view = this.getCurrentFocus(); - if (view != null) { - InputMethodManager imm = (InputMethodManager) getSystemService(Context.INPUT_METHOD_SERVICE); - imm.hideSoftInputFromWindow(view.getWindowToken(), 0); - } - } - - private void prepareDemoAssets() { - // We render from raw data with openGL, so disable decoding preprocessing - BitmapFactory.Options decodeOptions = new BitmapFactory.Options(); - decodeOptions.inScaled = false; - decodeOptions.inDither = false; - decodeOptions.inPremultiplied = false; - - try { - InputStream inputStream = getAssets().open(DEFAULT_GIF_TEXTURE); - defaultGIFTexture = - flipHorizontal( - BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions)); - inputStream.close(); - } catch (Exception e) { - Log.e(TAG, "Error parsing object texture; error: ", e); - throw new IllegalStateException(e); - } - - try { - InputStream inputStream = getAssets().open(ASSET_3D_TEXTURE); - asset3dTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions); - inputStream.close(); - } catch (Exception e) { - Log.e(TAG, "Error parsing object texture; error: ", e); - throw new IllegalStateException(e); - } - } - - private class MediaPipePacketManager implements FrameProcessor.OnWillAddFrameListener { - @Override - public void onWillAddFrame(long timestamp) { - // set current GIF bitmap as default texture - Bitmap currentGIFBitmap = defaultGIFTexture; - // If current index is in bounds, display current frame - if (gifCurrentIndex <= gifBitmaps.size() - 1) { - currentGIFBitmap = gifBitmaps.get(gifCurrentIndex); - } - // Update to next GIF frame based on timing and frame rate - updateGIFFrame(); - - // Calculate and set the aspect ratio of the GIF - float gifAspectRatio = - (float) currentGIFBitmap.getWidth() / (float) currentGIFBitmap.getHeight(); - - Packet stickerSentinelPacket = processor.getPacketCreator().createInt32(stickerSentinel); - // Sticker sentinel value must be reset for next graph iteration - stickerSentinel = -1; - // Initialize sticker data protobufferpacket information - Packet stickerProtoDataPacket = - processor - .getPacketCreator() - .createSerializedProto(StickerManager.getMessageLiteData(stickerArrayList)); - // Define and set the IMU sensory information float array - Packet imuDataPacket = processor.getPacketCreator().createFloat32Array(rotationMatrix); - // Communicate GIF textures (dynamic texturing) to graph - Packet gifTexturePacket = processor.getPacketCreator().createRgbaImageFrame(currentGIFBitmap); - Packet gifAspectRatioPacket = processor.getPacketCreator().createFloat32(gifAspectRatio); - processor - .getGraph() - .addConsumablePacketToInputStream(STICKER_SENTINEL_TAG, stickerSentinelPacket, timestamp); - processor - .getGraph() - .addConsumablePacketToInputStream(STICKER_PROTO_TAG, stickerProtoDataPacket, timestamp); - processor - .getGraph() - .addConsumablePacketToInputStream(IMU_MATRIX_TAG, imuDataPacket, timestamp); - processor - .getGraph() - .addConsumablePacketToInputStream(GIF_TEXTURE_TAG, gifTexturePacket, timestamp); - processor - .getGraph() - .addConsumablePacketToInputStream(GIF_ASPECT_RATIO_TAG, gifAspectRatioPacket, timestamp); - stickerSentinelPacket.release(); - stickerProtoDataPacket.release(); - imuDataPacket.release(); - gifTexturePacket.release(); - gifAspectRatioPacket.release(); - } - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/StickerManager.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/StickerManager.java deleted file mode 100644 index e6da53624..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/StickerManager.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.instantmotiontracking; - -import com.google.mediapipe.graphs.instantmotiontracking.StickerBufferProto.Sticker; -import com.google.mediapipe.graphs.instantmotiontracking.StickerBufferProto.StickerRoll; -import java.util.ArrayList; - -/** - * This class represents a single sticker object placed in the - * instantmotiontracking system. StickerManagers represent a unique object to render - * and manipulate in an AR scene. - *

A sticker has a sticker_id (a unique integer identifying a sticker object - * to render), x and y normalized anchor coordinates [0.0-1.0], user inputs for - * rotation in radians, scaling, and a renderID (another unique integer which - * determines what object model to render for this unique sticker). - */ -public class StickerManager { - - /** All types of possible objects to render for our application. */ - public enum Render { - // Every possible render for a sticker object - GIF, - ASSET_3D; - - /** - * Once called, will set the value of the current render to the next - * possible Render available. If all possible Renders have been iterated - * through, the function will loop and set to the first available Render. - */ - public Render iterate() { - int newEnumIdx = (this.ordinal() + 1) % Render.values().length; - return Render.values()[newEnumIdx]; - } - } - - // Current render of the sticker object - private Render currentRender; - - // Normalized X and Y coordinates of anchor - // (0,0) lies at top-left corner of screen - // (1.0,1.0) lies at bottom-right corner of screen - private float anchorX; - private float anchorY; - - // Rotation in radians from user - private float userRotation = 0f; - // Scaling factor as defined by user (defaults to 1.0) - private float userScalingFactor = 1f; - - // Unique sticker integer ID - private final int stickerId; - - // Used to determine next stickerId - private static int globalIDLimit = 1; - - /** - * Used to create a StickerManager object with a newly generated stickerId and a - * default Render of the first possible render in our Render enum. - */ - public StickerManager() { - // Every sticker will have a default render of the first 3D asset - this.currentRender = Render.values()[1]; - // StickerManager will render out of view by default - this.setAnchorCoordinate(2.0f, 2.0f); - // Set the global sticker ID limit for the next sticker - stickerId = StickerManager.globalIDLimit++; - } - - /** - * Used to create a StickerManager object with a newly generated stickerId. - * - * @param render initial Render of the new StickerManager object - */ - public StickerManager(Render render) { - this.currentRender = render; - // StickerManager will render out of view by default - this.setAnchorCoordinate(2.0f, 2.0f); - // Set the global sticker ID limit for the next sticker - stickerId = StickerManager.globalIDLimit++; - } - - /** - * Used to get the sticker ID of the object. - * - * @return integer of the unique sticker ID - */ - public int getstickerId() { - return this.stickerId; - } - - /** - * Used to update or reset the anchor positions in normalized [0.0-1.0] - * coordinate space for the sticker object. - * - * @param normalizedX normalized X coordinate for the new anchor position - * @param normalizedY normalized Y coordinate for the new anchor position - */ - public void setAnchorCoordinate(float normalizedX, float normalizedY) { - this.anchorX = normalizedX; - this.anchorY = normalizedY; - } - - /** Returns the normalized X anchor coordinate of the sticker object. */ - public float getAnchorX() { - return anchorX; - } - - /** Returns the normalized Y anchor coordinate of the sticker object. */ - public float getAnchorY() { - return anchorY; - } - - /** Returns current asset to be rendered for this sticker object. */ - public Render getRender() { - return currentRender; - } - - /** Set render for this sticker object */ - public void setRender(Render render) { - this.currentRender = render; - } - - /** - * Sets new user value of rotation radians. This rotation is not cumulative, - * and must be set to an absolute value of rotation applied to the object. - * - * @param radians specified radians to rotate the sticker object by - */ - public void setRotation(float radians) { - this.userRotation = radians; - } - - /** Returns current user radian rotation setting. */ - public float getRotation() { - return this.userRotation; - } - - /** - * Sets new user scale factor. This factor will be proportional to the scale - * of the sticker object. - * - * @param scaling scale factor to be applied - */ - public void setScaleFactor(float scaling) { - this.userScalingFactor = scaling; - } - - /** Returns current user scale factor setting. */ - public float getScaleFactor() { - return this.userScalingFactor; - } - - /** - * This method converts an ArrayList of stickers to a MessageLite object - * which can be passed directly to the MediaPipe graph. - * - * @param stickerArrayList ArrayList of StickerManager objects to convert to data string - * @return MessageLite protobuffer of all sticker data - */ - public static StickerRoll getMessageLiteData( - ArrayList stickerArrayList) { - StickerRoll.Builder stickerRollBuilder - = StickerRoll.newBuilder(); - for (final StickerManager sticker : stickerArrayList) { - Sticker protoSticker = - Sticker.newBuilder() - .setId(sticker.getstickerId()) - .setX(sticker.getAnchorX()) - .setY(sticker.getAnchorY()) - .setRotation(sticker.getRotation()) - .setScale(sticker.getScaleFactor()) - .setRenderId(sticker.getRender().ordinal()) - .build(); - stickerRollBuilder.addSticker(protoSticker); - } - return stickerRollBuilder.build(); - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/BUILD deleted file mode 100644 index e60b04c30..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/default_gif_texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/default_gif_texture.jpg deleted file mode 100644 index 27e86f96e..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/default_gif_texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/gif.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/gif.obj.uuu deleted file mode 100644 index 6e63ae6a5..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/gif/gif.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot.obj.uuu.zip b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot.obj.uuu.zip deleted file mode 100644 index 00a753dd1..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot.obj.uuu.zip and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot_texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot_texture.jpg deleted file mode 100644 index f41e41a6f..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/assets/robot/robot_texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_3d_preview.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_3d_preview.png deleted file mode 100644 index a1242817a..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_3d_preview.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_gif_preview.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_gif_preview.xml deleted file mode 100644 index fe7758dd9..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/asset_gif_preview.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_add_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_add_24.xml deleted file mode 100644 index eb232541d..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_add_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_arrow_back_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_arrow_back_24.xml deleted file mode 100644 index bab545a70..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_arrow_back_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_24.xml deleted file mode 100644 index 16d6d37dd..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_all_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_all_24.xml deleted file mode 100644 index dc649f3e2..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_clear_all_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_loop_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_loop_24.xml deleted file mode 100644 index c2f773a17..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_loop_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_search_24.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_search_24.xml deleted file mode 100644 index 07b76d627..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/baseline_search_24.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/circle_button.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/circle_button.xml deleted file mode 100644 index 067ae1f8b..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/drawable/circle_button.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/layout/instant_motion_tracking_activity_main.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/layout/instant_motion_tracking_activity_main.xml deleted file mode 100644 index c99a3c4a4..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/instantmotiontracking/res/layout/instant_motion_tracking_activity_main.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD deleted file mode 100644 index f629951df..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/BUILD +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "iristrackinggpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/iris_tracking:iris_tracking_gpu.binarypb", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/iris_landmark:iris_landmark.tflite", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.iristrackinggpu", - "appName": "Iris Tracking", - "mainActivity": ".MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "iris_tracking_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "@com_google_protobuf//:protobuf_javalite", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/MainActivity.java deleted file mode 100644 index 8079daa75..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/iristrackinggpu/MainActivity.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.iristrackinggpu; - -import android.graphics.SurfaceTexture; -import android.os.Bundle; -import android.util.Log; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; -import com.google.mediapipe.framework.Packet; -import com.google.mediapipe.framework.PacketGetter; -import com.google.protobuf.InvalidProtocolBufferException; -import java.util.HashMap; -import java.util.Map; - -/** Main activity of MediaPipe iris tracking app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - private static final String FOCAL_LENGTH_STREAM_NAME = "focal_length_pixel"; - private static final String OUTPUT_LANDMARKS_STREAM_NAME = "face_landmarks_with_iris"; - - private boolean haveAddedSidePackets = false; - - @Override - protected void onCameraStarted(SurfaceTexture surfaceTexture) { - super.onCameraStarted(surfaceTexture); - - // onCameraStarted gets called each time the activity resumes, but we only want to do this once. - if (!haveAddedSidePackets) { - float focalLength = cameraHelper.getFocalLengthPixels(); - if (focalLength != Float.MIN_VALUE) { - Packet focalLengthSidePacket = processor.getPacketCreator().createFloat32(focalLength); - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put(FOCAL_LENGTH_STREAM_NAME, focalLengthSidePacket); - processor.setInputSidePackets(inputSidePackets); - } - haveAddedSidePackets = true; - } - } - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - // To show verbose logging, run: - // adb shell setprop log.tag.MainActivity VERBOSE - if (Log.isLoggable(TAG, Log.VERBOSE)) { - processor.addPacketCallback( - OUTPUT_LANDMARKS_STREAM_NAME, - (packet) -> { - byte[] landmarksRaw = PacketGetter.getProtoBytes(packet); - try { - NormalizedLandmarkList landmarks = NormalizedLandmarkList.parseFrom(landmarksRaw); - if (landmarks == null) { - Log.v(TAG, "[TS:" + packet.getTimestamp() + "] No landmarks."); - return; - } - Log.v( - TAG, - "[TS:" - + packet.getTimestamp() - + "] #Landmarks for face (including iris): " - + landmarks.getLandmarkCount()); - Log.v(TAG, getLandmarksDebugString(landmarks)); - } catch (InvalidProtocolBufferException e) { - Log.e(TAG, "Couldn't Exception received - " + e); - return; - } - }); - } - } - - private static String getLandmarksDebugString(NormalizedLandmarkList landmarks) { - int landmarkIndex = 0; - String landmarksString = ""; - for (NormalizedLandmark landmark : landmarks.getLandmarkList()) { - landmarksString += - "\t\tLandmark[" - + landmarkIndex - + "]: (" - + landmark.getX() - + ", " - + landmark.getY() - + ", " - + landmark.getZ() - + ")\n"; - ++landmarkIndex; - } - return landmarksString; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD deleted file mode 100644 index 783ae200e..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("@bazel_skylib//lib:selects.bzl", "selects") -load(":build_defs.bzl", "generate_manifest_values") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -config_setting( - name = "use_chair", - define_values = { - "chair": "true", - }, -) - -config_setting( - name = "use_cup", - define_values = { - "cup": "true", - }, -) - -config_setting( - name = "use_camera", - define_values = { - "camera": "true", - }, -) - -config_setting( - name = "use_shoe_1stage", - define_values = { - "shoe_1stage": "true", - }, -) - -config_setting( - name = "use_chair_1stage", - define_values = { - "chair_1stage": "true", - }, -) - -selects.config_setting_group( - name = "1stage", - match_any = [ - ":use_shoe_1stage", - ":use_chair_1stage", - ], -) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = select({ - "//conditions:default": ["//mediapipe/graphs/object_detection_3d:mobile_calculators"], - ":1stage": ["//mediapipe/graphs/object_detection_3d:mobile_calculators_1stage"], - }) + [ - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -genrule( - name = "binary_graph", - srcs = select({ - "//conditions:default": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_binary_graph"], - ":1stage": ["//mediapipe/graphs/object_detection_3d:mobile_gpu_1stage_binary_graph"], - }), - outs = ["object_detection_3d.binarypb"], - cmd = "cp $< $@", -) - -MODELS_DIR = "//mediapipe/modules/objectron" - -genrule( - name = "model", - srcs = select({ - "//conditions:default": [MODELS_DIR + ":object_detection_3d_sneakers.tflite"], - ":use_chair": [MODELS_DIR + ":object_detection_3d_chair.tflite"], - ":use_cup": [MODELS_DIR + ":object_detection_3d_cup.tflite"], - ":use_camera": [MODELS_DIR + ":object_detection_3d_camera.tflite"], - ":use_shoe_1stage": [MODELS_DIR + ":object_detection_3d_sneakers_1stage.tflite"], - ":use_chair_1stage": [MODELS_DIR + ":object_detection_3d_chair_1stage.tflite"], - }), - outs = ["object_detection_3d.tflite"], - cmd = "cp $< $@", -) - -MANIFESTS_DIR = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests" - -android_library( - name = "manifest_lib", - exports_manifest = 1, - manifest = select({ - "//conditions:default": MANIFESTS_DIR + ":AndroidManifestSneaker.xml", - ":use_chair": MANIFESTS_DIR + ":AndroidManifestChair.xml", - ":use_cup": MANIFESTS_DIR + ":AndroidManifestCup.xml", - ":use_camera": MANIFESTS_DIR + ":AndroidManifestCamera.xml", - ":use_shoe_1stage": MANIFESTS_DIR + ":AndroidManifestSneaker.xml", - ":use_chair_1stage": MANIFESTS_DIR + ":AndroidManifestChair.xml", - }), - deps = [ - "//third_party:opencv", - "@maven//:androidx_concurrent_concurrent_futures", - "@maven//:com_google_guava_guava", - ], -) - -ASSETS_DIR = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets" - -genrule( - name = "mesh", - srcs = select({ - "//conditions:default": [ASSETS_DIR + "/sneaker:model.obj.uuu"], - ":use_chair": [ASSETS_DIR + "/chair:model.obj.uuu"], - ":use_cup": [ASSETS_DIR + "/cup:model.obj.uuu"], - ":use_camera": [ASSETS_DIR + "/camera:model.obj.uuu"], - ":use_shoe_1stage": [ASSETS_DIR + "/sneaker:model.obj.uuu"], - ":use_chair_1stage": [ASSETS_DIR + "/chair:model.obj.uuu"], - }), - outs = ["model.obj.uuu"], - cmd = "cp $< $@", -) - -genrule( - name = "texture", - srcs = select({ - "//conditions:default": [ASSETS_DIR + "/sneaker:texture.jpg"], - ":use_chair": [ASSETS_DIR + "/chair:texture.jpg"], - ":use_cup": [ASSETS_DIR + "/cup:texture.jpg"], - ":use_camera": [ASSETS_DIR + "/camera:texture.jpg"], - ":use_shoe_1stage": [ASSETS_DIR + "/sneaker:texture.jpg"], - ":use_chair_1stage": [ASSETS_DIR + "/chair:texture.jpg"], - }), - outs = ["texture.jpg"], - cmd = "cp $< $@", -) - -android_binary( - name = "objectdetection3d", - srcs = glob(["*.java"]), - assets = [ - ":binary_graph", - ":model", - ":mesh", - ":texture", - MODELS_DIR + ":object_detection_ssd_mobilenetv2_oidv4_fp16.tflite", - MODELS_DIR + ":object_detection_oidv4_labelmap.txt", - ASSETS_DIR + ":box.obj.uuu", - ASSETS_DIR + ":classic_colors.png", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = select({ - "//conditions:default": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_shoe", "Shoe Objectron"), - ":use_chair": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_chair", "Chair Objectron"), - ":use_cup": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_cup", "Cup Objectron"), - ":use_camera": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_camera", "Camera Objectron"), - ":use_shoe_1stage": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_shoe_1stage", "Single Stage Shoe Objectron"), - ":use_chair_1stage": generate_manifest_values("com.google.mediapipe.apps.objectdetection3d_chair_1stage", "Single Stage Chair Objectron"), - }), - multidex = "native", - deps = [ - ":manifest_lib", - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/MainActivity.java deleted file mode 100644 index b3a6dfeea..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/MainActivity.java +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.objectdetection3d; - -import android.content.pm.ApplicationInfo; -import android.content.pm.PackageManager; -import android.content.pm.PackageManager.NameNotFoundException; -import android.graphics.Bitmap; -import android.graphics.BitmapFactory; -import android.os.Bundle; -import android.util.Log; -import android.util.Size; -import android.view.SurfaceHolder; -import com.google.mediapipe.framework.AndroidAssetUtil; -import com.google.mediapipe.framework.AndroidPacketCreator; -import com.google.mediapipe.framework.Packet; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; - -/** Main activity of MediaPipe object detection 3D app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - private static final String OBJ_TEXTURE = "texture.jpg"; - private static final String OBJ_FILE = "model.obj.uuu"; - private static final String BOX_TEXTURE = "classic_colors.png"; - private static final String BOX_FILE = "box.obj.uuu"; - - // Assets. - private Bitmap objTexture = null; - private Bitmap boxTexture = null; - - // ApplicationInfo for retrieving metadata defined in the manifest. - private ApplicationInfo applicationInfo; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - try { - applicationInfo = - getPackageManager().getApplicationInfo(getPackageName(), PackageManager.GET_META_DATA); - } catch (NameNotFoundException e) { - Log.e(TAG, "Cannot find application info: " + e); - } - // Get allowed object category. - String categoryName = applicationInfo.metaData.getString("categoryName"); - // Get maximum allowed number of objects. - int maxNumObjects = applicationInfo.metaData.getInt("maxNumObjects"); - float[] modelScale = parseFloatArrayFromString( - applicationInfo.metaData.getString("modelScale")); - float[] modelTransform = parseFloatArrayFromString( - applicationInfo.metaData.getString("modelTransformation")); - prepareDemoAssets(); - AndroidPacketCreator packetCreator = processor.getPacketCreator(); - Map inputSidePackets = new HashMap<>(); - inputSidePackets.put("obj_asset_name", packetCreator.createString(OBJ_FILE)); - inputSidePackets.put("box_asset_name", packetCreator.createString(BOX_FILE)); - inputSidePackets.put("obj_texture", packetCreator.createRgbaImageFrame(objTexture)); - inputSidePackets.put("box_texture", packetCreator.createRgbaImageFrame(boxTexture)); - inputSidePackets.put("allowed_labels", packetCreator.createString(categoryName)); - inputSidePackets.put("max_num_objects", packetCreator.createInt32(maxNumObjects)); - inputSidePackets.put("model_scale", packetCreator.createFloat32Array(modelScale)); - inputSidePackets.put("model_transformation", packetCreator.createFloat32Array(modelTransform)); - processor.setInputSidePackets(inputSidePackets); - } - - @Override - protected Size cameraTargetResolution() { - return new Size(1280, 960); // Prefer 4:3 aspect ratio (camera size is in landscape). - } - - @Override - protected Size computeViewSize(int width, int height) { - return new Size(height, height * 3 / 4); // Prefer 3:4 aspect ratio. - } - - @Override - protected void onPreviewDisplaySurfaceChanged( - SurfaceHolder holder, int format, int width, int height) { - super.onPreviewDisplaySurfaceChanged(holder, format, width, height); - - boolean isCameraRotated = cameraHelper.isCameraRotated(); - Size cameraImageSize = cameraHelper.getFrameSize(); - processor.setOnWillAddFrameListener( - (timestamp) -> { - try { - int cameraTextureWidth = - isCameraRotated ? cameraImageSize.getHeight() : cameraImageSize.getWidth(); - int cameraTextureHeight = - isCameraRotated ? cameraImageSize.getWidth() : cameraImageSize.getHeight(); - - // Find limiting side and scale to 3:4 aspect ratio - float aspectRatio = (float) cameraTextureWidth / (float) cameraTextureHeight; - if (aspectRatio > 3.0 / 4.0) { - // width too big - cameraTextureWidth = (int) ((float) cameraTextureHeight * 3.0 / 4.0); - } else { - // height too big - cameraTextureHeight = (int) ((float) cameraTextureWidth * 4.0 / 3.0); - } - Packet widthPacket = processor.getPacketCreator().createInt32(cameraTextureWidth); - Packet heightPacket = processor.getPacketCreator().createInt32(cameraTextureHeight); - - try { - processor.getGraph().addPacketToInputStream("input_width", widthPacket, timestamp); - processor.getGraph().addPacketToInputStream("input_height", heightPacket, timestamp); - } catch (RuntimeException e) { - Log.e( - TAG, - "MediaPipeException encountered adding packets to input_width and input_height" - + " input streams.", e); - } - widthPacket.release(); - heightPacket.release(); - } catch (IllegalStateException ise) { - Log.e(TAG, "Exception while adding packets to width and height input streams."); - } - }); - } - - private void prepareDemoAssets() { - AndroidAssetUtil.initializeNativeAssetManager(this); - // We render from raw data with openGL, so disable decoding preprocessing - BitmapFactory.Options decodeOptions = new BitmapFactory.Options(); - decodeOptions.inScaled = false; - decodeOptions.inDither = false; - decodeOptions.inPremultiplied = false; - - try { - InputStream inputStream = getAssets().open(OBJ_TEXTURE); - objTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions); - inputStream.close(); - } catch (Exception e) { - Log.e(TAG, "Error parsing object texture; error: " + e); - throw new IllegalStateException(e); - } - - try { - InputStream inputStream = getAssets().open(BOX_TEXTURE); - boxTexture = BitmapFactory.decodeStream(inputStream, null /*outPadding*/, decodeOptions); - inputStream.close(); - } catch (Exception e) { - Log.e(TAG, "Error parsing box texture; error: " + e); - throw new RuntimeException(e); - } - } - - private static float[] parseFloatArrayFromString(String string) { - String[] elements = string.split(",", -1); - float[] array = new float[elements.length]; - for (int i = 0; i < elements.length; ++i) { - array[i] = Float.parseFloat(elements[i]); - } - return array; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/box.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/box.obj.uuu deleted file mode 100644 index 80a2aa8ca..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/box.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/model.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/model.obj.uuu deleted file mode 100644 index 0280d5dd0..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/model.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/texture.jpg deleted file mode 100644 index 4a19534dd..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/camera/texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/model.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/model.obj.uuu deleted file mode 100644 index e2fc9bc1f..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/model.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/texture.jpg deleted file mode 100644 index 759172f5c..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/chair/texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/classic_colors.png b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/classic_colors.png deleted file mode 100644 index 92dad8ef6..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/classic_colors.png and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/colors.bmp b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/colors.bmp deleted file mode 100644 index 1bbb1ca07..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/colors.bmp and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/model.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/model.obj.uuu deleted file mode 100644 index 167e134eb..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/model.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/texture.jpg deleted file mode 100644 index f3aea3568..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/cup/texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/model.obj.uuu b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/model.obj.uuu deleted file mode 100644 index ee5183652..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/model.obj.uuu and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/texture.jpg b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/texture.jpg deleted file mode 100644 index 58f641bfe..000000000 Binary files a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/assets/sneaker/texture.jpg and /dev/null differ diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/build_defs.bzl b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/build_defs.bzl deleted file mode 100644 index 9c30dd58c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/build_defs.bzl +++ /dev/null @@ -1,15 +0,0 @@ -"""Build defs for Objectron.""" - -def generate_manifest_values(application_id, app_name): - manifest_values = { - "applicationId": application_id, - "appName": app_name, - "mainActivity": "com.google.mediapipe.apps.objectdetection3d.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "object_detection_3d.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - } - return manifest_values diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCamera.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCamera.xml deleted file mode 100644 index 10f8492ef..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCamera.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestChair.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestChair.xml deleted file mode 100644 index cd6502fa2..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestChair.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCup.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCup.xml deleted file mode 100644 index a06694563..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestCup.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestSneaker.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestSneaker.xml deleted file mode 100644 index a1fd1143b..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/AndroidManifestSneaker.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/BUILD deleted file mode 100644 index a8bb9124c..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/manifests/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files( - srcs = glob(["**"]), -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD deleted file mode 100644 index 9bb054936..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/object_detection:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "objectdetectioncpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/object_detection:mobile_cpu.binarypb", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.objectdetectioncpu", - "appName": "Object Detection (CPU)", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "mobile_cpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD deleted file mode 100644 index 81f2ed3e6..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/object_detection:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "objectdetectiongpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/object_detection:mobile_gpu.binarypb", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.objectdetectiongpu", - "appName": "Object Detection", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD deleted file mode 100644 index 50ea70f89..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/tracking:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "objecttrackinggpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/tracking:mobile_gpu.binarypb", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.objecttrackinggpu", - "appName": "Object Tracking", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "mobile_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD deleted file mode 100644 index d1c45345f..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/BUILD +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/pose_tracking:pose_tracking_gpu_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "posetrackinggpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/pose_tracking:pose_tracking_gpu.binarypb", - "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", - "//mediapipe/modules/pose_detection:pose_detection.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.posetrackinggpu", - "appName": "Pose Tracking", - "mainActivity": ".MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "pose_tracking_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - "//mediapipe/framework/formats:landmark_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "@com_google_protobuf//:protobuf_javalite", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/MainActivity.java deleted file mode 100644 index 730aa6e1f..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackinggpu/MainActivity.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.apps.posetrackinggpu; - -import android.os.Bundle; -import android.util.Log; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; -import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; -import com.google.mediapipe.framework.PacketGetter; -import com.google.protobuf.InvalidProtocolBufferException; - -/** Main activity of MediaPipe pose tracking app. */ -public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { - private static final String TAG = "MainActivity"; - - private static final String OUTPUT_LANDMARKS_STREAM_NAME = "pose_landmarks"; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - - // To show verbose logging, run: - // adb shell setprop log.tag.MainActivity VERBOSE - if (Log.isLoggable(TAG, Log.VERBOSE)) { - processor.addPacketCallback( - OUTPUT_LANDMARKS_STREAM_NAME, - (packet) -> { - Log.v(TAG, "Received pose landmarks packet."); - try { - NormalizedLandmarkList poseLandmarks = - PacketGetter.getProto(packet, NormalizedLandmarkList.class); - Log.v( - TAG, - "[TS:" - + packet.getTimestamp() - + "] " - + getPoseLandmarksDebugString(poseLandmarks)); - } catch (InvalidProtocolBufferException exception) { - Log.e(TAG, "Failed to get proto.", exception); - } - }); - } - } - - private static String getPoseLandmarksDebugString(NormalizedLandmarkList poseLandmarks) { - String poseLandmarkStr = "Pose landmarks: " + poseLandmarks.getLandmarkCount() + "\n"; - int landmarkIndex = 0; - for (NormalizedLandmark landmark : poseLandmarks.getLandmarkList()) { - poseLandmarkStr += - "\tLandmark [" - + landmarkIndex - + "]: (" - + landmark.getX() - + ", " - + landmark.getY() - + ", " - + landmark.getZ() - + ")\n"; - ++landmarkIndex; - } - return poseLandmarkStr; - } -} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD deleted file mode 100644 index 6bfcf34c1..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/selfiesegmentationgpu/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu_deps", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "selfiesegmentationgpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu.binarypb", - "//mediapipe/modules/selfie_segmentation:selfie_segmentation.tflite", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.selfiesegmentationgpu", - "appName": "Selfie Segmentation", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "True", - "binaryGraphName": "selfie_segmentation_gpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD deleted file mode 100644 index ed3a63a70..000000000 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/templatematchingcpu/BUILD +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -cc_binary( - name = "libmediapipe_jni.so", - linkshared = 1, - linkstatic = 1, - deps = [ - "//mediapipe/graphs/template_matching:mobile_calculators", - "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", - ], -) - -cc_library( - name = "mediapipe_jni_lib", - srcs = [":libmediapipe_jni.so"], - alwayslink = 1, -) - -android_binary( - name = "templatematchingcpu", - srcs = glob(["*.java"]), - assets = [ - "//mediapipe/graphs/template_matching:mobile_cpu.binarypb", - "//mediapipe/models:knift_index.pb", - "//mediapipe/models:knift_float.tflite", - "//mediapipe/models:knift_labelmap.txt", - ], - assets_dir = "", - manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.apps.templatematchingcpu", - "appName": "Template Matching", - "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", - "cameraFacingFront": "False", - "binaryGraphName": "mobile_cpu.binarypb", - "inputVideoStreamName": "input_video", - "outputVideoStreamName": "output_video", - "flipFramesVertically": "True", - "converterNumBuffers": "2", - }, - multidex = "native", - deps = [ - ":mediapipe_jni_lib", - "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", - ], -) diff --git a/mediapipe/examples/coral/BUILD b/mediapipe/examples/coral/BUILD deleted file mode 100644 index 50f0b38c7..000000000 --- a/mediapipe/examples/coral/BUILD +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = [ - "//visibility:public", -]) - -# Graph Runner - -cc_library( - name = "demo_run_graph_main", - srcs = ["demo_run_graph_main.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - ], -) - -# Demos - -cc_binary( - name = "object_detection_tpu", - deps = [ - "//mediapipe/examples/coral:demo_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_tflite_calculators", - ], -) - -cc_binary( - name = "face_detection_tpu", - deps = [ - "//mediapipe/examples/coral:demo_run_graph_main", - "//mediapipe/graphs/face_detection:desktop_live_calculators", - ], -) diff --git a/mediapipe/examples/coral/Dockerfile b/mediapipe/examples/coral/Dockerfile deleted file mode 100644 index ea99e5b08..000000000 --- a/mediapipe/examples/coral/Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#==== ! Prerequisite ! ==== -# $ sh mediapipe/examples/coral/setup.sh -#==== - -# for opencv 3.2 default -FROM ubuntu:18.04 - -MAINTAINER - -WORKDIR /mediapipe - -ENV DEBIAN_FRONTEND=noninteractive - -# Install MediaPipe & Coral deps - -COPY update_sources.sh / -RUN /update_sources.sh - -RUN dpkg --add-architecture armhf -RUN dpkg --add-architecture arm64 -RUN apt-get update && apt-get install -y \ - build-essential \ - crossbuild-essential-arm64 \ - libusb-1.0-0-dev:arm64 \ - zlibc:arm64 \ - pkg-config \ - zip \ - unzip \ - curl \ - wget \ - git \ - python \ - python-pip \ - python3-pip \ - python-numpy \ - vim-common \ - ca-certificates \ - emacs \ - software-properties-common && \ - add-apt-repository -y ppa:openjdk-r/ppa && \ - apt-get update && apt-get install -y openjdk-8-jdk - -RUN pip install --upgrade setuptools -RUN pip install future -RUN pip3 install six - -COPY . /mediapipe/ - -# Install bazel -# Please match the current MediaPipe Bazel requirements according to docs. -ARG BAZEL_VERSION=3.7.2 -RUN mkdir /bazel && \ - wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ - wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ - chmod +x /bazel/installer.sh && \ - /bazel/installer.sh && \ - rm -f /bazel/installer.sh - -# OpenCV (3.2 default in 18.04) - -RUN apt-get update && apt-get install -y libopencv-dev - -# Opencv libs copied from coral device into opencv32_arm64_libs - -RUN cp opencv32_arm64_libs/* /usr/lib/aarch64-linux-gnu/. - -# Edge tpu header and lib - -RUN git clone https://github.com/google-coral/edgetpu.git /edgetpu -RUN cp /edgetpu/libedgetpu/direct/aarch64/libedgetpu.so.1.0 /usr/lib/aarch64-linux-gnu/libedgetpu.so - -# See mediapipe/examples/coral/README.md to finish setup diff --git a/mediapipe/examples/coral/README.md b/mediapipe/examples/coral/README.md deleted file mode 100644 index a65f449bb..000000000 --- a/mediapipe/examples/coral/README.md +++ /dev/null @@ -1,156 +0,0 @@ -# Coral Dev Board Setup (experimental) - -**Disclaimer**: Running MediaPipe on Coral is experimental, and this process may -not be exact and is subject to change. These instructions have only been tested -on the [Coral Dev Board](https://coral.ai/products/dev-board/) -running [Mendel Enterprise Day 13](https://coral.ai/software/) OS and -using [Diploria2](https://github.com/google-coral/edgetpu/tree/diploria2) -edgetpu libs, and may vary for different devices and workstations. - -This file describes how to prepare a Coral Dev Board and setup a Linux -Docker container for building MediaPipe applications that run on Edge TPU. - -## Before creating the Docker - -* (on host machine) run _setup.sh_ from MediaPipe root directory - - sh mediapipe/examples/coral/setup.sh - -* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works - - Note: alias mdt="python3 -m mdt.main" may be needed on some systems - -* (on coral device) prepare MediaPipe - - cd ~ - sudo apt-get update && sudo apt-get install -y git - git clone https://github.com/google/mediapipe.git - mkdir mediapipe/bazel-bin - -* (on coral device) install opencv 3.2 - - sudo apt-get update && sudo apt-get install -y libopencv-dev - -* (on coral device) find all opencv libs - - find /usr/lib/aarch64-linux-gnu/ -name 'libopencv*so' - -* (on host machine) copy core opencv libs from coral device to a local folder inside MediaPipe checkout: - - # in root level mediapipe folder # - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_core.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_calib3d.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_features2d.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_highgui.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgcodecs.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgproc.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_video.so opencv32_arm64_libs - mdt pull /usr/lib/aarch64-linux-gnu/libopencv_videoio.so opencv32_arm64_libs - -* (on host machine) Create and start the docker environment - - # from mediapipe root level directory # - docker build -t coral . - docker run -it --name coral coral:latest - -## Inside the Docker environment - -* Update library paths in /mediapipe/third_party/opencv_linux.BUILD - - (replace 'x86_64-linux-gnu' with 'aarch64-linux-gnu') - - "lib/aarch64-linux-gnu/libopencv_core.so", - "lib/aarch64-linux-gnu/libopencv_calib3d.so", - "lib/aarch64-linux-gnu/libopencv_features2d.so", - "lib/aarch64-linux-gnu/libopencv_highgui.so", - "lib/aarch64-linux-gnu/libopencv_imgcodecs.so", - "lib/aarch64-linux-gnu/libopencv_imgproc.so", - "lib/aarch64-linux-gnu/libopencv_video.so", - "lib/aarch64-linux-gnu/libopencv_videoio.so", - -* Attempt to build hello world (to download external deps) - - bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world - -* Edit /edgetpu/libedgetpu/BUILD - - to add this build target - - cc_library( - name = "lib", - srcs = [ - "libedgetpu.so", - ], - visibility = ["//visibility:public"], - ) - -* Edit /edgetpu/WORKSPACE - - update /mediapipe/WORKSPACE TENSORFLOW_* variables to match what /edgetpu/WORKSPACE has: - - grep TENSORFLOW_ /mediapipe/WORKSPACE - grep TENSORFLOW_ /edgetpu/WORKSPACE - - # Make sure the /mediapipe/WORKSPACE _TENSORFLOW_GIT_COMMIT and _TENSORFLOW_SHA256 - # match the /edgetpu/WORKSPACE TENSORFLOW_COMMIT and TENSORFLOW_SHA256 respectively. - - # If they do not match, modify /mediapipe/WORKSPACE to match what /edgetpu/WORKSPACE has. - # Also comment out the MediaPipe org_tensorflow patch section. - -* Edit /mediapipe/mediapipe/calculators/tflite/BUILD to change rules for *tflite_inference_calculator.cc* - - sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' /mediapipe/mediapipe/calculators/tflite/BUILD - - The above command should add - - "@edgetpu//:header", - "@libedgetpu//:lib", - - to the _deps_ of tflite_inference_calculator.cc - - Now also remove XNNPACK deps: - - sed -i 's/\"@org_tensorflow\/\/tensorflow\/lite\/delegates\/xnnpack/#\"@org_tensorflow\/\/tensorflow\/lite\/delegates\/xnnpack/g' /mediapipe/mediapipe/calculators/tflite/BUILD - -#### Now try cross-compiling for device - -* Object detection demo - -![Object Detection running on Coral](./images/object_detection_demo_coral.jpg) - - bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_tpu - - Copy object_detection_tpu binary to the MediaPipe checkout on the coral device - - # outside docker env, open new terminal on host machine # - docker ps - docker cp :/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_tpu /tmp/. - mdt push /tmp/object_detection_tpu /home/mendel/mediapipe/bazel-bin/. - -* Face detection demo - -![Face Detection running on Coral](./images/face_detection_demo_coral.gif) - - bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_tpu - - Copy face_detection_tpu binary to the MediaPipe checkout on the coral device - - # outside docker env, open new terminal on host machine # - docker ps - docker cp :/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_tpu /tmp/. - mdt push /tmp/face_detection_tpu /home/mendel/mediapipe/bazel-bin/. - -## On the coral device (with display) - - # Object detection - cd ~/mediapipe - chmod +x bazel-bin/object_detection_tpu - export GLOG_logtostderr=1 - bazel-bin/object_detection_tpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt - - # Face detection - cd ~/mediapipe - chmod +x bazel-bin/face_detection_tpu - export GLOG_logtostderr=1 - bazel-bin/face_detection_tpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt - diff --git a/mediapipe/examples/coral/WORKSPACE.coral b/mediapipe/examples/coral/WORKSPACE.coral deleted file mode 100644 index f990ef2f5..000000000 --- a/mediapipe/examples/coral/WORKSPACE.coral +++ /dev/null @@ -1,30 +0,0 @@ - -### Coral additions to MediaPipe WORKSPACE ### - -#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}') -#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}') -# Oct 2019 -#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229 -#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb -http_archive( - name = "coral_crosstool", - sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb", - strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229", - urls = [ - "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz", - ], -) -load("@coral_crosstool//:configure.bzl", "cc_crosstool") -cc_crosstool(name = "crosstool") - -# EdgeTPU -new_local_repository( - name = "edgetpu", - path = "/edgetpu/libedgetpu", - build_file = "/edgetpu/libedgetpu/BUILD" -) -new_local_repository( - name = "libedgetpu", - path = "/usr/lib/aarch64-linux-gnu", - build_file = "/edgetpu/libedgetpu/BUILD" -) diff --git a/mediapipe/examples/coral/demo_run_graph_main.cc b/mediapipe/examples/coral/demo_run_graph_main.cc deleted file mode 100644 index 6f1c56268..000000000 --- a/mediapipe/examples/coral/demo_run_graph_main.cc +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// An example of sending OpenCV webcam frames into a MediaPipe graph. -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -constexpr char kInputStream[] = "input_video"; -constexpr char kOutputStream[] = "output_video"; -constexpr char kWindowName[] = "MediaPipe"; - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); -ABSL_FLAG(std::string, input_video_path, "", - "Full path of video to load. " - "If not provided, attempt to use a webcam."); -ABSL_FLAG(std::string, output_video_path, "", - "Full path of where to save result (.mp4 only). " - "If not provided, show result in a window."); - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config)); - - LOG(INFO) << "Initialize the camera or load the video."; - cv::VideoCapture capture; - const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); - if (load_video) { - capture.open(absl::GetFlag(FLAGS_input_video_path)); - } else { - capture.open(0); - } - RET_CHECK(capture.isOpened()); - - cv::VideoWriter writer; - const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty(); - if (save_video) { - LOG(INFO) << "Prepare video writer."; - cv::Mat test_frame; - capture.read(test_frame); // Consume first frame. - capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. - writer.open(absl::GetFlag(FLAGS_output_video_path), - mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 - capture.get(cv::CAP_PROP_FPS), test_frame.size()); - RET_CHECK(writer.isOpened()); - } else { - cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); - capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); - capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); - capture.set(cv::CAP_PROP_AUTOFOCUS, 0); - capture.set(cv::CAP_PROP_FOCUS, 1); - capture.set(cv::CAP_PROP_FPS, 30); - } - - LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, - graph.AddOutputStreamPoller(kOutputStream)); - MP_RETURN_IF_ERROR(graph.StartRun({})); - - LOG(INFO) << "Start grabbing and processing frames."; - bool grab_frames = true; - while (grab_frames) { - // Capture opencv camera or video frame. - cv::Mat camera_frame_raw; - capture >> camera_frame_raw; - if (camera_frame_raw.empty()) break; // End of video. - cv::Mat camera_frame; - cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); - if (!load_video) { - cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - } - - // Wrap Mat into an ImageFrame. - auto input_frame = absl::make_unique( - mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, - mediapipe::ImageFrame::kDefaultAlignmentBoundary); - cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); - camera_frame.copyTo(input_frame_mat); - - // Send image packet into the graph. - size_t frame_timestamp_us = - (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; - MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( - kInputStream, mediapipe::Adopt(input_frame.release()) - .At(mediapipe::Timestamp(frame_timestamp_us)))); - - // Get the graph result packet, or stop if that fails. - mediapipe::Packet packet; - if (!poller.Next(&packet)) break; - auto& output_frame = packet.Get(); - - // Convert back to opencv for display or saving. - cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); - cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); - if (save_video) { - writer.write(output_frame_mat); - } else { - cv::imshow(kWindowName, output_frame_mat); - // Press any key to exit. - const int pressed_key = cv::waitKey(5); - if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; - } - } - - LOG(INFO) << "Shutting down."; - if (writer.isOpened()) writer.release(); - MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); - return graph.WaitUntilDone(); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt deleted file mode 100644 index fe72b14d6..000000000 --- a/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt +++ /dev/null @@ -1,176 +0,0 @@ -# MediaPipe graph that performs face detection with TensorFlow Lite on TPU. -# Used in the examples in -# mediapipe/examples/coral:face_detection_tpu. - -# Images on GPU coming into and out of the graph. -input_stream: "input_video" -output_stream: "output_video" - -# Throttles the images flowing downstream for flow control. It passes through -# the very first incoming image unaltered, and waits for -# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish -# generating the corresponding detections before it passes through another -# image. All images that come in while waiting are dropped, limiting the number -# of in-flight images between this calculator and -# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between -# from queuing up incoming images and data excessively, which leads to increased -# latency and memory usage, unwanted in real-time mobile applications. It also -# eliminates unnecessarily computation, e.g., a transformed image produced by -# ImageTransformationCalculator may get dropped downstream if the subsequent -# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy -# processing previous inputs. -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:detections" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -# Transforms the input image on CPU to a 128x128 image. To scale the input -# image, the scale_mode option is set to FIT to preserve the aspect ratio, -# resulting in potential letterboxing in the transformed image. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:throttled_input_video" - output_stream: "IMAGE:transformed_input_video" - output_stream: "LETTERBOX_PADDING:letterbox_padding" - options: { - [mediapipe.ImageTransformationCalculatorOptions.ext] { - output_width: 128 - output_height: 128 - scale_mode: FIT - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" - options: { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - use_quantized_tensors: true - } - } -} - -# Runs a TensorFlow Lite model on TPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - options: { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options: { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 4 - min_scale: 0.1484375 - max_scale: 0.75 - input_size_height: 128 - input_size_width: 128 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 8 - strides: 16 - strides: 16 - strides: 16 - aspect_ratios: 1.0 - fixed_anchor_size: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { - num_classes: 1 - num_boxes: 896 - num_coords: 16 - box_coord_offset: 0 - keypoint_coord_offset: 4 - num_keypoints: 6 - num_values_per_keypoint: 2 - sigmoid_score: true - score_clipping_thresh: 100.0 - reverse_output_order: true - x_scale: 128.0 - y_scale: 128.0 - h_scale: 128.0 - w_scale: 128.0 - min_score_thresh: 0.75 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - options: { - [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { - min_suppression_threshold: 0.3 - overlap_type: INTERSECTION_OVER_UNION - algorithm: WEIGHTED - return_empty_detections: true - } - } -} - -# Adjusts detection locations (already normalized to [0.f, 1.f]) on the -# letterboxed image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (the -# input image to the graph before image transformation). -node { - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:filtered_detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:output_detections" -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - options: { - [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:throttled_input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} - diff --git a/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt deleted file mode 100644 index 03bc9e18f..000000000 --- a/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt +++ /dev/null @@ -1,179 +0,0 @@ -# MediaPipe graph that performs object detection with TensorFlow Lite on TPU. -# Used in the examples in -# mediapipie/examples/coral:object_detection_tpu. - -# Images on TPU coming into and out of the graph. -input_stream: "input_video" -output_stream: "output_video" - -# Throttles the images flowing downstream for flow control. It passes through -# the very first incoming image unaltered, and waits for -# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish -# generating the corresponding detections before it passes through another -# image. All images that come in while waiting are dropped, limiting the number -# of in-flight images between this calculator and -# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between -# from queuing up incoming images and data excessively, which leads to increased -# latency and memory usage, unwanted in real-time mobile applications. It also -# eliminates unnecessarily computation, e.g., a transformed image produced by -# ImageTransformationCalculator may get dropped downstream if the subsequent -# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy -# processing previous inputs. -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:detections" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -# Transforms the input image on CPU to a 300x300 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:throttled_input_video" - output_stream: "IMAGE:transformed_input_video" - options: { - [mediapipe.ImageTransformationCalculatorOptions.ext] { - output_width: 300 - output_height: 300 - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" - options: { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - use_quantized_tensors: true - } - } -} - -# Runs a TensorFlow Lite model on TPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - options: { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options: { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 300 - input_size_width: 300 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - sigmoid_score: true - apply_exponential_on_box_size: true - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - min_score_thresh: 0.6 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - options: { - [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { - min_suppression_threshold: 0.4 - max_num_detections: 3 - overlap_type: INTERSECTION_OVER_UNION - return_empty_detections: true - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - options: { - [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] { - label_map_path: "mediapipe/examples/coral/models/object_detection_labelmap.txt" - } - } -} - -# Converts the detections to drawing primitives for annotation overlay. -node { - calculator: "DetectionsToRenderDataCalculator" - input_stream: "DETECTIONS:output_detections" - output_stream: "RENDER_DATA:render_data" - options: { - [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] { - thickness: 4.0 - color { r: 255 g: 0 b: 0 } - } - } -} - -# Draws annotations and overlays them on top of the input images. -node { - calculator: "AnnotationOverlayCalculator" - input_stream: "IMAGE:throttled_input_video" - input_stream: "render_data" - output_stream: "IMAGE:output_video" -} diff --git a/mediapipe/examples/coral/images/face_detection_demo_coral.gif b/mediapipe/examples/coral/images/face_detection_demo_coral.gif deleted file mode 100644 index d006bb000..000000000 Binary files a/mediapipe/examples/coral/images/face_detection_demo_coral.gif and /dev/null differ diff --git a/mediapipe/examples/coral/images/object_detection_demo_coral.jpg b/mediapipe/examples/coral/images/object_detection_demo_coral.jpg deleted file mode 100644 index 8ceb4a1a7..000000000 Binary files a/mediapipe/examples/coral/images/object_detection_demo_coral.jpg and /dev/null differ diff --git a/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite b/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite deleted file mode 100644 index 2f2b4b55e..000000000 Binary files a/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite and /dev/null differ diff --git a/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite b/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite deleted file mode 100644 index 7e6cf12f6..000000000 Binary files a/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite and /dev/null differ diff --git a/mediapipe/examples/coral/models/object_detection_labelmap.txt b/mediapipe/examples/coral/models/object_detection_labelmap.txt deleted file mode 100644 index 695772dcd..000000000 --- a/mediapipe/examples/coral/models/object_detection_labelmap.txt +++ /dev/null @@ -1,90 +0,0 @@ -person -bicycle -car -motorcycle -airplane -bus -train -truck -boat -traffic light -fire hydrant -??? -stop sign -parking meter -bench -bird -cat -dog -horse -sheep -cow -elephant -bear -zebra -giraffe -??? -backpack -umbrella -??? -??? -handbag -tie -suitcase -frisbee -skis -snowboard -sports ball -kite -baseball bat -baseball glove -skateboard -surfboard -tennis racket -bottle -??? -wine glass -cup -fork -knife -spoon -bowl -banana -apple -sandwich -orange -broccoli -carrot -hot dog -pizza -donut -cake -chair -couch -potted plant -bed -??? -dining table -??? -??? -toilet -??? -tv -laptop -mouse -remote -keyboard -cell phone -microwave -oven -toaster -sink -refrigerator -??? -book -clock -vase -scissors -teddy bear -hair drier -toothbrush diff --git a/mediapipe/examples/coral/setup.sh b/mediapipe/examples/coral/setup.sh deleted file mode 100755 index 884a7a10d..000000000 --- a/mediapipe/examples/coral/setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -set -e -set -v - -echo 'Please run this from root level mediapipe directory! \n Ex:' -echo ' sh mediapipe/examples/coral/setup.sh ' - -sleep 3 - -mkdir -p opencv32_arm64_libs - -# prepare docker aux script -cp mediapipe/examples/coral/update_sources.sh update_sources.sh -chmod +x update_sources.sh - -# backup non-coral Dockerfile -mv Dockerfile Dockerfile.orig -cp mediapipe/examples/coral/Dockerfile Dockerfile - -# backup non-coral workspace -cp WORKSPACE WORKSPACE.orig - -# create temps -cp WORKSPACE WORKSPACE.1 -cp mediapipe/examples/coral/WORKSPACE.coral WORKSPACE.2 - -# merge (shell decides concat order, unless numbered appropriately) -cat WORKSPACE.1 WORKSPACE.2 > WORKSPACE - -# cleanup -rm WORKSPACE.1 WORKSPACE.2 - -echo 'done' diff --git a/mediapipe/examples/coral/update_sources.sh b/mediapipe/examples/coral/update_sources.sh deleted file mode 100755 index dcd336e67..000000000 --- a/mediapipe/examples/coral/update_sources.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# To run in the Coral Docker environment. - -. /etc/os-release - -sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list - -echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe" >> /etc/apt/sources.list -echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe" >> /etc/apt/sources.list -echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe" >> /etc/apt/sources.list diff --git a/mediapipe/examples/desktop/BUILD b/mediapipe/examples/desktop/BUILD deleted file mode 100644 index 80cb7ad81..000000000 --- a/mediapipe/examples/desktop/BUILD +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = [ - "//visibility:public", -]) - -cc_library( - name = "simple_run_graph_main", - srcs = ["simple_run_graph_main.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "demo_run_graph_main", - srcs = ["demo_run_graph_main.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - ], -) - -# Linux only. -# Must have a GPU with EGL support: -# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev -# (or similar nvidia/amd equivalent) -cc_library( - name = "demo_run_graph_main_gpu", - srcs = ["demo_run_graph_main_gpu.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:gpu_shared_data_internal", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - ], -) diff --git a/mediapipe/examples/desktop/README.md b/mediapipe/examples/desktop/README.md deleted file mode 100644 index 6880098ba..000000000 --- a/mediapipe/examples/desktop/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains MediaPipe example applications for desktop. Please see [Solutions](https://solutions.mediapipe.dev)for details. diff --git a/mediapipe/examples/desktop/__init__.py b/mediapipe/examples/desktop/__init__.py deleted file mode 100644 index 6db73bc52..000000000 --- a/mediapipe/examples/desktop/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Copyright 2019 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/mediapipe/examples/desktop/autoflip/BUILD b/mediapipe/examples/desktop/autoflip/BUILD deleted file mode 100644 index 9d84e2bdb..000000000 --- a/mediapipe/examples/desktop/autoflip/BUILD +++ /dev/null @@ -1,58 +0,0 @@ -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -proto_library( - name = "autoflip_messages_proto", - srcs = ["autoflip_messages.proto"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "autoflip_messages_cc_proto", - srcs = ["autoflip_messages.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = [ - "//mediapipe/examples:__subpackages__", - ], - deps = [":autoflip_messages_proto"], -) - -cc_binary( - name = "run_autoflip", - deps = [ - "//mediapipe/calculators/core:packet_thinner_calculator", - "//mediapipe/calculators/image:scale_image_calculator", - "//mediapipe/calculators/video:opencv_video_decoder_calculator", - "//mediapipe/calculators/video:opencv_video_encoder_calculator", - "//mediapipe/calculators/video:video_pre_stream_calculator", - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/examples/desktop/autoflip/calculators:border_detection_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:face_to_region_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:localization_to_region_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:scene_cropping_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:shot_boundary_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:signal_fusing_calculator", - "//mediapipe/examples/desktop/autoflip/calculators:video_filtering_calculator", - "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_face_detection_subgraph", - "//mediapipe/examples/desktop/autoflip/subgraph:autoflip_object_detection_subgraph", - ], -) diff --git a/mediapipe/examples/desktop/autoflip/README.md b/mediapipe/examples/desktop/autoflip/README.md deleted file mode 100644 index 98004a782..000000000 --- a/mediapipe/examples/desktop/autoflip/README.md +++ /dev/null @@ -1,25 +0,0 @@ -### Steps to run the AutoFlip video cropping graph - -1. Checkout the repository and follow - [the installation instructions](https://github.com/google/mediapipe/blob/master/mediapipe/docs/install.md) - to set up MediaPipe. - - ```bash - git clone https://github.com/google/mediapipe.git - cd mediapipe - ``` - -2. Build and run the run_autoflip binary to process a local video. - -Note: AutoFlip currently only works with OpenCV 3 . Please verify your OpenCV version beforehand. - - ```bash - bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \ - mediapipe/examples/desktop/autoflip:run_autoflip - - GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/autoflip/run_autoflip \ - --calculator_graph_config_file=mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt \ - --input_side_packets=input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/absolute/path/to/save/the/output/video/file,aspect_ratio=width:height - ``` - -3. View the cropped video. diff --git a/mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt b/mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt deleted file mode 100644 index 95af18815..000000000 --- a/mediapipe/examples/desktop/autoflip/autoflip_graph.pbtxt +++ /dev/null @@ -1,202 +0,0 @@ -# Autoflip graph that only renders the final cropped video. For use with -# end user applications. -max_queue_size: -1 - -# VIDEO_PREP: Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:video_raw" - output_stream: "VIDEO_PRESTREAM:video_header" - output_side_packet: "SAVED_AUDIO_PATH:audio_path" -} - -# VIDEO_PREP: Scale the input video before feature extraction. -node { - calculator: "ScaleImageCalculator" - input_stream: "FRAMES:video_raw" - input_stream: "VIDEO_HEADER:video_header" - output_stream: "FRAMES:video_frames_scaled" - options: { - [mediapipe.ScaleImageCalculatorOptions.ext]: { - preserve_aspect_ratio: true - output_format: SRGB - target_width: 480 - algorithm: DEFAULT_WITHOUT_UPSCALE - } - } -} - -# VIDEO_PREP: Create a low frame rate stream for feature extraction. -node { - calculator: "PacketThinnerCalculator" - input_stream: "video_frames_scaled" - output_stream: "video_frames_scaled_downsampled" - options: { - [mediapipe.PacketThinnerCalculatorOptions.ext]: { - thinner_type: ASYNC - period: 200000 - } - } -} - -# DETECTION: find borders around the video and major background color. -node { - calculator: "BorderDetectionCalculator" - input_stream: "VIDEO:video_raw" - output_stream: "DETECTED_BORDERS:borders" -} - -# DETECTION: find shot/scene boundaries on the full frame rate stream. -node { - calculator: "ShotBoundaryCalculator" - input_stream: "VIDEO:video_frames_scaled" - output_stream: "IS_SHOT_CHANGE:shot_change" - options { - [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] { - min_shot_span: 0.2 - min_motion: 0.3 - window_size: 15 - min_shot_measure: 10 - min_motion_with_shot_measure: 0.05 - } - } -} - -# DETECTION: find faces on the down sampled stream -node { - calculator: "AutoFlipFaceDetectionSubgraph" - input_stream: "VIDEO:video_frames_scaled_downsampled" - output_stream: "DETECTIONS:face_detections" -} -node { - calculator: "FaceToRegionCalculator" - input_stream: "VIDEO:video_frames_scaled_downsampled" - input_stream: "FACES:face_detections" - output_stream: "REGIONS:face_regions" -} - -# DETECTION: find objects on the down sampled stream -node { - calculator: "AutoFlipObjectDetectionSubgraph" - input_stream: "VIDEO:video_frames_scaled_downsampled" - output_stream: "DETECTIONS:object_detections" -} -node { - calculator: "LocalizationToRegionCalculator" - input_stream: "DETECTIONS:object_detections" - output_stream: "REGIONS:object_regions" - options { - [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] { - output_all_signals: true - } - } -} - -# SIGNAL FUSION: Combine detections (with weights) on each frame -node { - calculator: "SignalFusingCalculator" - input_stream: "shot_change" - input_stream: "face_regions" - input_stream: "object_regions" - output_stream: "salient_regions" - options { - [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] { - signal_settings { - type { standard: FACE_CORE_LANDMARKS } - min_score: 0.85 - max_score: 0.9 - is_required: false - } - signal_settings { - type { standard: FACE_ALL_LANDMARKS } - min_score: 0.8 - max_score: 0.85 - is_required: false - } - signal_settings { - type { standard: FACE_FULL } - min_score: 0.8 - max_score: 0.85 - is_required: false - } - signal_settings { - type: { standard: HUMAN } - min_score: 0.75 - max_score: 0.8 - is_required: false - } - signal_settings { - type: { standard: PET } - min_score: 0.7 - max_score: 0.75 - is_required: false - } - signal_settings { - type: { standard: CAR } - min_score: 0.7 - max_score: 0.75 - is_required: false - } - signal_settings { - type: { standard: OBJECT } - min_score: 0.1 - max_score: 0.2 - is_required: false - } - } - } -} - -# CROPPING: make decisions about how to crop each frame. -node { - calculator: "SceneCroppingCalculator" - input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio" - input_stream: "VIDEO_FRAMES:video_raw" - input_stream: "KEY_FRAMES:video_frames_scaled_downsampled" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:borders" - input_stream: "SHOT_BOUNDARIES:shot_change" - output_stream: "CROPPED_FRAMES:cropped_frames" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - max_scene_size: 600 - key_frame_crop_options: { - score_aggregation_type: CONSTANT - } - scene_camera_motion_analyzer_options: { - motion_stabilization_threshold_percent: 0.5 - salient_point_bound: 0.499 - } - padding_parameters: { - blur_cv_size: 200 - overlay_opacity: 0.6 - } - target_size_type: MAXIMIZE_TARGET_DIMENSION - } - } -} - -# ENCODING(required): encode the video stream for the final cropped output. -node { - calculator: "VideoPreStreamCalculator" - # Fetch frame format and dimension from input frames. - input_stream: "FRAME:cropped_frames" - # Copying frame rate and duration from original video. - input_stream: "VIDEO_PRESTREAM:video_header" - output_stream: "output_frames_video_header" -} - -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:cropped_frames" - input_stream: "VIDEO_PRESTREAM:output_frames_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - input_side_packet: "AUDIO_FILE_PATH:audio_path" - options: { - [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt b/mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt deleted file mode 100644 index 0087cd690..000000000 --- a/mediapipe/examples/desktop/autoflip/autoflip_graph_development.pbtxt +++ /dev/null @@ -1,252 +0,0 @@ -# Autoflip graph that renders the final cropped video and debugging videos. -# For use by developers who may be adding signals and adjusting weights. -max_queue_size: -1 - -# VIDEO_PREP: Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:video_raw" - output_stream: "VIDEO_PRESTREAM:video_header" - output_side_packet: "SAVED_AUDIO_PATH:audio_path" -} - -# VIDEO_PREP: Scale the input video before feature extraction. -node { - calculator: "ScaleImageCalculator" - input_stream: "FRAMES:video_raw" - input_stream: "VIDEO_HEADER:video_header" - output_stream: "FRAMES:video_frames_scaled" - options: { - [mediapipe.ScaleImageCalculatorOptions.ext]: { - preserve_aspect_ratio: true - output_format: SRGB - target_width: 480 - algorithm: DEFAULT_WITHOUT_UPSCALE - } - } -} - -# VIDEO_PREP: Create a low frame rate stream for feature extraction. -node { - calculator: "PacketThinnerCalculator" - input_stream: "video_frames_scaled" - output_stream: "video_frames_scaled_downsampled" - options: { - [mediapipe.PacketThinnerCalculatorOptions.ext]: { - thinner_type: ASYNC - period: 200000 - } - } -} - -# DETECTION: find borders around the video and major background color. -node { - calculator: "BorderDetectionCalculator" - input_stream: "VIDEO:video_raw" - output_stream: "DETECTED_BORDERS:borders" -} - -# DETECTION: find shot/scene boundaries on the full frame rate stream. -node { - calculator: "ShotBoundaryCalculator" - input_stream: "VIDEO:video_frames_scaled" - output_stream: "IS_SHOT_CHANGE:shot_change" - options { - [mediapipe.autoflip.ShotBoundaryCalculatorOptions.ext] { - min_shot_span: 0.2 - min_motion: 0.3 - window_size: 15 - min_shot_measure: 10 - min_motion_with_shot_measure: 0.05 - } - } -} - -# DETECTION: find faces on the down sampled stream -node { - calculator: "AutoFlipFaceDetectionSubgraph" - input_stream: "VIDEO:video_frames_scaled_downsampled" - output_stream: "DETECTIONS:face_detections" -} -node { - calculator: "FaceToRegionCalculator" - input_stream: "VIDEO:video_frames_scaled_downsampled" - input_stream: "FACES:face_detections" - output_stream: "REGIONS:face_regions" -} - -# DETECTION: find objects on the down sampled stream -node { - calculator: "AutoFlipObjectDetectionSubgraph" - input_stream: "VIDEO:video_frames_scaled_downsampled" - output_stream: "DETECTIONS:object_detections" -} -node { - calculator: "LocalizationToRegionCalculator" - input_stream: "DETECTIONS:object_detections" - output_stream: "REGIONS:object_regions" - options { - [mediapipe.autoflip.LocalizationToRegionCalculatorOptions.ext] { - output_all_signals: true - } - } -} - -# SIGNAL FUSION: Combine detections (with weights) on each frame -node { - calculator: "SignalFusingCalculator" - input_stream: "shot_change" - input_stream: "face_regions" - input_stream: "object_regions" - output_stream: "salient_regions" - options { - [mediapipe.autoflip.SignalFusingCalculatorOptions.ext] { - signal_settings { - type { standard: FACE_CORE_LANDMARKS } - min_score: 0.85 - max_score: 0.9 - is_required: false - } - signal_settings { - type { standard: FACE_ALL_LANDMARKS } - min_score: 0.8 - max_score: 0.85 - is_required: false - } - signal_settings { - type { standard: FACE_FULL } - min_score: 0.8 - max_score: 0.85 - is_required: false - } - signal_settings { - type: { standard: HUMAN } - min_score: 0.75 - max_score: 0.8 - is_required: false - } - signal_settings { - type: { standard: PET } - min_score: 0.7 - max_score: 0.75 - is_required: false - } - signal_settings { - type: { standard: CAR } - min_score: 0.7 - max_score: 0.75 - is_required: false - } - signal_settings { - type: { standard: OBJECT } - min_score: 0.1 - max_score: 0.2 - is_required: false - } - } - } -} - -# CROPPING: make decisions about how to crop each frame. -node { - calculator: "SceneCroppingCalculator" - input_side_packet: "EXTERNAL_ASPECT_RATIO:aspect_ratio" - input_stream: "VIDEO_FRAMES:video_raw" - input_stream: "KEY_FRAMES:video_frames_scaled_downsampled" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:borders" - input_stream: "SHOT_BOUNDARIES:shot_change" - output_stream: "CROPPED_FRAMES:cropped_frames" - output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames" - output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - max_scene_size: 600 - key_frame_crop_options: { - score_aggregation_type: CONSTANT - } - scene_camera_motion_analyzer_options: { - motion_stabilization_threshold_percent: 0.5 - salient_point_bound: 0.499 - } - padding_parameters: { - blur_cv_size: 200 - overlay_opacity: 0.6 - } - target_size_type: MAXIMIZE_TARGET_DIMENSION - } - } -} - -# ENCODING(required): encode the video stream for the final cropped output. -node { - calculator: "VideoPreStreamCalculator" - # Fetch frame format and dimension from input frames. - input_stream: "FRAME:cropped_frames" - # Copying frame rate and duration from original video. - input_stream: "VIDEO_PRESTREAM:video_header" - output_stream: "output_frames_video_header" -} - -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:cropped_frames" - input_stream: "VIDEO_PRESTREAM:output_frames_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - input_side_packet: "AUDIO_FILE_PATH:audio_path" - options: { - [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: { - codec: "avc1" - video_format: "mp4" - } - } -} - -# ENCODING(optional): encode the video stream for the key_frame_crop_viz_frames -# output. Draws boxes around required and non-required objects. -node { - calculator: "VideoPreStreamCalculator" - # Fetch frame format and dimension from input frames. - input_stream: "FRAME:key_frame_crop_viz_frames" - # Copying frame rate and duration from original video. - input_stream: "VIDEO_PRESTREAM:video_header" - output_stream: "key_frame_crop_viz_frames_header" -} - -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:key_frame_crop_viz_frames" - input_stream: "VIDEO_PRESTREAM:key_frame_crop_viz_frames_header" - input_side_packet: "OUTPUT_FILE_PATH:key_frame_crop_viz_frames_path" - options: { - [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: { - codec: "avc1" - video_format: "mp4" - } - } -} - -# ENCODING(optional): encode the video stream for the salient_point_viz_frames -# output. Draws the focus points and the scene crop window (red). -node { - calculator: "VideoPreStreamCalculator" - # Fetch frame format and dimension from input frames. - input_stream: "FRAME:salient_point_viz_frames" - # Copying frame rate and duration from original video. - input_stream: "VIDEO_PRESTREAM:video_header" - output_stream: "salient_point_viz_frames_header" -} - -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:salient_point_viz_frames" - input_stream: "VIDEO_PRESTREAM:salient_point_viz_frames_header" - input_side_packet: "OUTPUT_FILE_PATH:salient_point_viz_frames_path" - options: { - [mediapipe.OpenCvVideoEncoderCalculatorOptions.ext]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/examples/desktop/autoflip/autoflip_messages.proto b/mediapipe/examples/desktop/autoflip/autoflip_messages.proto deleted file mode 100644 index 726237e6b..000000000 --- a/mediapipe/examples/desktop/autoflip/autoflip_messages.proto +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Proto messages used for the AutoFlip Pipeline. -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -// Borders detected on the frame as well as non-border color (if present). -// Next tag: 4 -message StaticFeatures { - // A list of the static parts for a frame. - repeated Border border = 1; - // The background color (only set if solid color). - optional Color solid_background = 2; - // Area of the image that is not a border. - optional Rect non_static_area = 3; -} - -// A static border area within the video. -// Next tag: 3 -message Border { - // Original location within the input frame. - optional Rect border_position = 1; - // Position for static area. - // Next tag: 3 - enum RelativePosition { - TOP = 1; - BOTTOM = 2; - } - // Top or bottom position. - optional RelativePosition relative_position = 2; -} - -// Rectangle (opencv format). -// Next tag: 5 -message Rect { - optional int32 x = 1; - optional int32 y = 2; - optional int32 width = 3; - optional int32 height = 4; -} - -// Color (RGB 8bit) -// Next tag: 4 -message Color { - optional int32 r = 1; - optional int32 g = 2; - optional int32 b = 3; -} - -// Rectangle (opencv format). -// Next tag: 5 -message RectF { - optional float x = 1; - optional float y = 2; - optional float width = 3; - optional float height = 4; -} - -// An image region of interest (eg a detected face or object), accompanied by an -// importance score. -// Next tag: 10 -message SalientRegion { - reserved 3; - // The bounding box for this region in the image. - optional Rect location = 1; - - // The bounding box for this region in the image normalized. - optional RectF location_normalized = 8; - - // A score indicating the importance of this region. - optional float score = 2; - - // A tracking id used to identify this region across video frames. Not always - // set. - optional int64 tracking_id = 4; - - // If true, this region is required to be present in the final video (eg it - // contains text that cannot be cropped). - optional bool is_required = 5 [default = false]; - - // Type of signal carried in this message. - optional SignalType signal_type = 6; - - // If true, object cannot move in the output window (e.g. text would look - // strange moving around). - // TODO: this feature is not implemented, remove proto message. - optional bool requires_static_location = 7 [default = false]; - - // When used with ContentZoomingCalculator, this flag can be set in the - // SignalFusingCalculator indicating that areas outside of these detections - // can be cropped from the frame. When no salient regions have this flag set - // true, no zooming is performed. When one or more salient regions have this - // flag set true, the max zoom value will be used that keeps all - // “only_required” detections within view. The ContentZoomingCalculator - // currently supports zooming by finding the size of non-salient top/bottom - // borders regions and provides this information to the - // SceneCroppingCalculator for reframing. - optional bool only_required = 9 [default = false]; -} - -// Stores the message type, including standard types (face, object) and custom -// types defined by a string id. -// Next tag: 3 -message SignalType { - enum StandardType { - UNSET = 0; - // Full face bounding boxed detected. - FACE_FULL = 1; - // Face landmarks for eyes, nose, chin only. - FACE_CORE_LANDMARKS = 2; - // All face landmarks (eyes, ears, nose, chin). - FACE_ALL_LANDMARKS = 3; - // A specific face landmark. - FACE_LANDMARK = 4; - HUMAN = 5; - CAR = 6; - PET = 7; - OBJECT = 8; - MOTION = 9; - TEXT = 10; - LOGO = 11; - USER_HINT = 12; - } - oneof Signal { - StandardType standard = 1; - string custom = 2; - } -} - -// Features extracted from a image. -// Next tag: 3 -message DetectionSet { - // Mask image showing pixel-wise values at a given location. - optional string encoded_mask = 1; - // List of rectangle detections. - repeated SalientRegion detections = 2; -} - -// General settings needed for multiple calculators. -message ConversionOptions { - extend mediapipe.CalculatorOptions { - optional ConversionOptions ext = 284806832; - } - // Target output width of the conversion. - optional int32 target_width = 1; - // Target output height of the conversion. - optional int32 target_height = 2; -} - -// Self-contained message that provides all needed information to render -// autoflip with an external renderer. One of these messages is required for -// each frame of the video. -message ExternalRenderFrame { - // Rectangle using opencv standard. - message Rect { - optional float x = 1; - optional float y = 2; - optional float width = 3; - optional float height = 4; - } - // RGB color [0...255] - message Color { - optional int32 r = 1; - optional int32 g = 2; - optional int32 b = 3; - } - // Rect that must be cropped out of the input frame. It is in the - // original dimensions of the input video. The first step to render this - // frame is to crop this rect from the input frame. - optional Rect crop_from_location = 1; - // The placement location where the above rect is placed on the output frame. - // This will always have the same aspect ratio as the above rect but scaling - // may be required. - optional Rect render_to_location = 2; - // If render_to_location is smaller than the output dimensions of the frame, - // fill the rest of the frame with this color. - optional Color padding_color = 3; - // Timestamp in microseconds of this frame. - optional uint64 timestamp_us = 4; - // Target width of the cropped video in pixels. |render_to_location| is - // relative to this dimension. - optional int32 target_width = 5; - // Target height of the cropped video in pixels. |render_to_location| is - // relative to this dimension. - optional int32 target_height = 6; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/BUILD b/mediapipe/examples/desktop/autoflip/calculators/BUILD deleted file mode 100644 index 7ab845cbb..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/BUILD +++ /dev/null @@ -1,509 +0,0 @@ -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = [ - "//mediapipe/examples:__subpackages__", - "//mediapipe/viz:__subpackages__", -]) - -cc_library( - name = "border_detection_calculator", - srcs = ["border_detection_calculator.cc"], - deps = [ - ":border_detection_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -proto_library( - name = "border_detection_calculator_proto", - srcs = ["border_detection_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "border_detection_calculator_cc_proto", - srcs = ["border_detection_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":border_detection_calculator_proto"], -) - -cc_library( - name = "content_zooming_calculator_state", - hdrs = ["content_zooming_calculator_state.h"], - deps = [ - "//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:rect_cc_proto", - ], -) - -cc_library( - name = "content_zooming_calculator", - srcs = ["content_zooming_calculator.cc"], - deps = [ - ":content_zooming_calculator_cc_proto", - ":content_zooming_calculator_state", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -proto_library( - name = "content_zooming_calculator_proto", - srcs = ["content_zooming_calculator.proto"], - deps = [ - "//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "content_zooming_calculator_cc_proto", - srcs = ["content_zooming_calculator.proto"], - cc_deps = [ - "//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = [ - "//mediapipe/examples:__subpackages__", - ], - deps = [ - ":content_zooming_calculator_proto", - ], -) - -cc_test( - name = "border_detection_calculator_test", - srcs = [ - "border_detection_calculator_test.cc", - ], - linkstatic = 1, - deps = [ - ":border_detection_calculator", - ":border_detection_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:benchmark", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "content_zooming_calculator_test", - srcs = [ - "content_zooming_calculator_test.cc", - ], - linkstatic = 1, - deps = [ - ":content_zooming_calculator", - ":content_zooming_calculator_cc_proto", - ":content_zooming_calculator_state", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/formats:rect_cc_proto", - "//mediapipe/framework/port:benchmark", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "video_filtering_calculator", - srcs = ["video_filtering_calculator.cc"], - copts = ["-fexceptions"], - features = ["-use_header_modules"], # Incompatible with -fexceptions. - deps = [ - ":video_filtering_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -proto_library( - name = "video_filtering_calculator_proto", - srcs = ["video_filtering_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "video_filtering_calculator_cc_proto", - srcs = ["video_filtering_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":video_filtering_calculator_proto"], -) - -cc_test( - name = "video_filtering_calculator_test", - srcs = ["video_filtering_calculator_test.cc"], - deps = [ - ":video_filtering_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -proto_library( - name = "scene_cropping_calculator_proto", - srcs = ["scene_cropping_calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/examples/desktop/autoflip/quality:cropping_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "scene_cropping_calculator_cc_proto", - srcs = ["scene_cropping_calculator.proto"], - cc_deps = [ - "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":scene_cropping_calculator_proto"], -) - -cc_library( - name = "scene_cropping_calculator", - srcs = ["scene_cropping_calculator.cc"], - hdrs = ["scene_cropping_calculator.h"], - deps = [ - ":scene_cropping_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/examples/desktop/autoflip/quality:cropping_cc_proto", - "//mediapipe/examples/desktop/autoflip/quality:focus_point_cc_proto", - "//mediapipe/examples/desktop/autoflip/quality:frame_crop_region_computer", - "//mediapipe/examples/desktop/autoflip/quality:padding_effect_generator", - "//mediapipe/examples/desktop/autoflip/quality:piecewise_linear_function", - "//mediapipe/examples/desktop/autoflip/quality:polynomial_regression_path_solver", - "//mediapipe/examples/desktop/autoflip/quality:scene_camera_motion_analyzer", - "//mediapipe/examples/desktop/autoflip/quality:scene_cropper", - "//mediapipe/examples/desktop/autoflip/quality:scene_cropping_viz", - "//mediapipe/examples/desktop/autoflip/quality:utils", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings:str_format", - ], - alwayslink = 1, # buildozer: disable=alwayslink-with-hdrs -) - -cc_test( - name = "scene_cropping_calculator_test", - size = "large", - timeout = "long", - srcs = ["scene_cropping_calculator_test.cc"], - deps = [ - ":scene_cropping_calculator", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "signal_fusing_calculator", - srcs = ["signal_fusing_calculator.cc"], - deps = [ - ":signal_fusing_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -proto_library( - name = "signal_fusing_calculator_proto", - srcs = ["signal_fusing_calculator.proto"], - deps = [ - "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "signal_fusing_calculator_cc_proto", - srcs = ["signal_fusing_calculator.proto"], - cc_deps = [ - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":signal_fusing_calculator_proto"], -) - -cc_test( - name = "signal_fusing_calculator_test", - srcs = ["signal_fusing_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":signal_fusing_calculator", - ":signal_fusing_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "shot_boundary_calculator", - srcs = ["shot_boundary_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":shot_boundary_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], - alwayslink = 1, -) - -proto_library( - name = "shot_boundary_calculator_proto", - srcs = ["shot_boundary_calculator.proto"], - deps = [ - "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "shot_boundary_calculator_cc_proto", - srcs = ["shot_boundary_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":shot_boundary_calculator_proto"], -) - -cc_test( - name = "shot_boundary_calculator_test", - srcs = ["shot_boundary_calculator_test.cc"], - data = ["//mediapipe/examples/desktop/autoflip/calculators/testdata:test_images"], - linkstatic = 1, - deps = [ - ":shot_boundary_calculator", - ":shot_boundary_calculator_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "face_to_region_calculator", - srcs = ["face_to_region_calculator.cc"], - deps = [ - ":face_to_region_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/examples/desktop/autoflip/quality:visual_scorer", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], - alwayslink = 1, -) - -proto_library( - name = "face_to_region_calculator_proto", - srcs = ["face_to_region_calculator.proto"], - deps = [ - "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "face_to_region_calculator_cc_proto", - srcs = ["face_to_region_calculator.proto"], - cc_deps = [ - "//mediapipe/examples/desktop/autoflip/quality:visual_scorer_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":face_to_region_calculator_proto"], -) - -cc_test( - name = "face_to_region_calculator_test", - srcs = ["face_to_region_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":face_to_region_calculator", - ":face_to_region_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -proto_library( - name = "localization_to_region_calculator_proto", - srcs = ["localization_to_region_calculator.proto"], - deps = [ - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "localization_to_region_calculator_cc_proto", - srcs = ["localization_to_region_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":localization_to_region_calculator_proto"], -) - -cc_library( - name = "localization_to_region_calculator", - srcs = ["localization_to_region_calculator.cc"], - visibility = ["//visibility:public"], - deps = [ - ":localization_to_region_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], - alwayslink = 1, -) - -cc_test( - name = "localization_to_region_calculator_test", - srcs = ["localization_to_region_calculator_test.cc"], - linkstatic = 1, - deps = [ - ":localization_to_region_calculator", - ":localization_to_region_calculator_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework:calculator_runner", - "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) diff --git a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc deleted file mode 100644 index caaa368a7..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.cc +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// This Calculator takes an ImageFrame and scales it appropriately. - -#include -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -using mediapipe::Adopt; -using mediapipe::CalculatorBase; -using mediapipe::ImageFrame; -using mediapipe::PacketTypeSet; -using mediapipe::autoflip::Border; - -constexpr char kDetectedBorders[] = "DETECTED_BORDERS"; -constexpr int kMinBorderDistance = 5; -constexpr int kKMeansClusterCount = 4; -constexpr int kMaxPixelsToProcess = 300000; -constexpr char kVideoInputTag[] = "VIDEO"; - -namespace mediapipe { -namespace autoflip { - -namespace { - -// Sets rect values into a proto. -void SetRect(const cv::Rect& region, - const Border::RelativePosition& relative_position, Border* part) { - part->mutable_border_position()->set_x(region.x); - part->mutable_border_position()->set_y(region.y); - part->mutable_border_position()->set_width(region.width); - part->mutable_border_position()->set_height(region.height); - part->set_relative_position(relative_position); -} - -} // namespace - -// This calculator takes a sequence of images (video) and detects solid color -// borders as well as the dominant color of the non-border area. This per-frame -// information is passed to downstream calculators. -class BorderDetectionCalculator : public CalculatorBase { - public: - BorderDetectionCalculator() : frame_width_(-1), frame_height_(-1) {} - ~BorderDetectionCalculator() override {} - BorderDetectionCalculator(const BorderDetectionCalculator&) = delete; - BorderDetectionCalculator& operator=(const BorderDetectionCalculator&) = - delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - - private: - // Given a color and image direction, check to see if a border of that color - // exists. - void DetectBorder(const cv::Mat& frame, const Color& color, - const Border::RelativePosition& direction, - StaticFeatures* features); - - // Provide the percent this color shows up in a given image. - double ColorCount(const Color& mask_color, const cv::Mat& image) const; - - // Set member vars (image size) and confirm no changes frame-to-frame. - absl::Status SetAndCheckInputs(const cv::Mat& frame); - - // Find the dominant color for a input image. - double FindDominantColor(const cv::Mat& image, Color* dominant_color); - - // Frame width and height. - int frame_width_; - int frame_height_; - - // Options for processing. - BorderDetectionCalculatorOptions options_; -}; -REGISTER_CALCULATOR(BorderDetectionCalculator); - -absl::Status BorderDetectionCalculator::Open(mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - RET_CHECK_LT(options_.vertical_search_distance(), 0.5) - << "Search distance must be less than half the full image."; - return absl::OkStatus(); -} - -absl::Status BorderDetectionCalculator::SetAndCheckInputs( - const cv::Mat& frame) { - if (frame_width_ < 0) { - frame_width_ = frame.cols; - } - if (frame_height_ < 0) { - frame_height_ = frame.rows; - } - RET_CHECK_EQ(frame.cols, frame_width_) - << "Input frame dimensions must remain constant throughout the video."; - RET_CHECK_EQ(frame.rows, frame_height_) - << "Input frame dimensions must remain constant throughout the video."; - RET_CHECK_EQ(frame.channels(), 3) << "Input video type must be 3-channel"; - return absl::OkStatus(); -} - -absl::Status BorderDetectionCalculator::Process( - mediapipe::CalculatorContext* cc) { - if (!cc->Inputs().HasTag(kVideoInputTag) || - cc->Inputs().Tag(kVideoInputTag).Value().IsEmpty()) { - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Input tag VIDEO not set or empty at timestamp: " - << cc->InputTimestamp().Value(); - } - cv::Mat frame = mediapipe::formats::MatView( - &cc->Inputs().Tag(kVideoInputTag).Get()); - MP_RETURN_IF_ERROR(SetAndCheckInputs(frame)); - - // Initialize output and set default values. - std::unique_ptr features = - absl::make_unique(); - features->mutable_non_static_area()->set_x(0); - features->mutable_non_static_area()->set_width(frame_width_); - features->mutable_non_static_area()->set_y(options_.default_padding_px()); - features->mutable_non_static_area()->set_height( - std::max(0, frame_height_ - options_.default_padding_px() * 2)); - - // Check for border at the top of the frame. - Color seed_color_top; - FindDominantColor(frame(cv::Rect(0, 0, frame_width_, 1)), &seed_color_top); - DetectBorder(frame, seed_color_top, Border::TOP, features.get()); - - // Check for border at the bottom of the frame. - Color seed_color_bottom; - FindDominantColor(frame(cv::Rect(0, frame_height_ - 1, frame_width_, 1)), - &seed_color_bottom); - DetectBorder(frame, seed_color_bottom, Border::BOTTOM, features.get()); - - // Check the non-border area for a dominant color. - cv::Mat non_static_frame = frame( - cv::Rect(features->non_static_area().x(), features->non_static_area().y(), - features->non_static_area().width(), - features->non_static_area().height())); - Color dominant_color_nonborder; - double dominant_color_percent = - FindDominantColor(non_static_frame, &dominant_color_nonborder); - if (dominant_color_percent > options_.solid_background_tol_perc()) { - auto* bg_color = features->mutable_solid_background(); - bg_color->set_r(dominant_color_nonborder.r()); - bg_color->set_g(dominant_color_nonborder.g()); - bg_color->set_b(dominant_color_nonborder.b()); - } - - // Output result. - cc->Outputs() - .Tag(kDetectedBorders) - .AddPacket(Adopt(features.release()).At(cc->InputTimestamp())); - - return absl::OkStatus(); -} - -// Find the dominant color within an image. -double BorderDetectionCalculator::FindDominantColor(const cv::Mat& image_raw, - Color* dominant_color) { - cv::Mat image; - if (image_raw.total() > kMaxPixelsToProcess) { - float resize = kMaxPixelsToProcess / static_cast(image_raw.total()); - cv::resize(image_raw, image, cv::Size(), resize, resize); - } else { - image = image_raw; - } - - cv::Mat float_data, cluster, cluster_center; - image.convertTo(float_data, CV_32F); - cv::Mat reshaped = float_data.reshape(1, float_data.total()); - - cv::kmeans(reshaped, kKMeansClusterCount, cluster, - cv::TermCriteria(CV_TERMCRIT_ITER, 5, 1.0), 1, - cv::KMEANS_PP_CENTERS, cluster_center); - - std::vector count(kKMeansClusterCount, 0); - for (int i = 0; i < cluster.rows; i++) { - count[cluster.at(i, 0)]++; - } - auto max_cluster_ptr = std::max_element(count.begin(), count.end()); - double max_cluster_perc = - *max_cluster_ptr / static_cast(cluster.rows); - int max_cluster_idx = std::distance(count.begin(), max_cluster_ptr); - - dominant_color->set_r(cluster_center.at(max_cluster_idx, 2)); - dominant_color->set_g(cluster_center.at(max_cluster_idx, 1)); - dominant_color->set_b(cluster_center.at(max_cluster_idx, 0)); - - return max_cluster_perc; -} - -double BorderDetectionCalculator::ColorCount(const Color& mask_color, - const cv::Mat& image) const { - int background_count = 0; - for (int i = 0; i < image.rows; i++) { - const uint8* row_ptr = image.ptr(i); - for (int j = 0; j < image.cols * 3; j += 3) { - if (std::abs(mask_color.r() - static_cast(row_ptr[j + 2])) <= - options_.color_tolerance() && - std::abs(mask_color.g() - static_cast(row_ptr[j + 1])) <= - options_.color_tolerance() && - std::abs(mask_color.b() - static_cast(row_ptr[j])) <= - options_.color_tolerance()) { - background_count++; - } - } - } - return background_count / static_cast(image.rows * image.cols); -} - -void BorderDetectionCalculator::DetectBorder( - const cv::Mat& frame, const Color& color, - const Border::RelativePosition& direction, StaticFeatures* features) { - // Search the entire image until we find an object, or hit the max search - // distance. - int search_distance = - (direction == Border::TOP || direction == Border::BOTTOM) ? frame.rows - : frame.cols; - search_distance *= options_.vertical_search_distance(); - - // Check if each next line has a dominant color that matches the given - // border color. - int last_border = -1; - for (int i = 0; i < search_distance; i++) { - cv::Rect current_row; - switch (direction) { - case Border::TOP: - current_row = cv::Rect(0, i, frame.cols, 1); - break; - case Border::BOTTOM: - current_row = cv::Rect(0, frame.rows - i - 1, frame.cols, 1); - break; - } - if (ColorCount(color, frame(current_row)) < - options_.border_color_pixel_perc()) { - break; - } - last_border = i; - } - - // Reject results that are not borders (or too small). - if (last_border <= kMinBorderDistance || last_border == search_distance - 1) { - return; - } - - // Apply defined padding. - last_border += options_.border_object_padding_px(); - - switch (direction) { - case Border::TOP: - SetRect(cv::Rect(0, 0, frame.cols, last_border), Border::TOP, - features->add_border()); - features->mutable_non_static_area()->set_y( - last_border + features->non_static_area().y()); - features->mutable_non_static_area()->set_height( - std::max(0, frame_height_ - (features->non_static_area().y() + - options_.default_padding_px()))); - break; - case Border::BOTTOM: - SetRect( - cv::Rect(0, frame.rows - last_border - 1, frame.cols, last_border), - Border::BOTTOM, features->add_border()); - - features->mutable_non_static_area()->set_height(std::max( - 0, frame.rows - (features->non_static_area().y() + last_border + - options_.default_padding_px()))); - - break; - } -} - -absl::Status BorderDetectionCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - cc->Inputs().Tag(kVideoInputTag).Set(); - cc->Outputs().Tag(kDetectedBorders).Set(); - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto deleted file mode 100644 index 9be74a3e4..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.proto +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -// Next tag: 7 -message BorderDetectionCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional BorderDetectionCalculatorOptions ext = 276599815; - } - // Max difference in color to be considered the same (per rgb channel). - optional int32 color_tolerance = 1 [default = 6]; - - // Amount of padding to add around any object within the border that is - // resized to fit into the new border. - optional int32 border_object_padding_px = 2 [default = 5]; - - // Distance (as a percent of height) to search for a border. - optional float vertical_search_distance = 3 [default = .20]; - - // Percent of pixels matching border color to be a border - optional float border_color_pixel_perc = 4 [default = .995]; - - // Percent of pixels matching background to be a solid background frame - optional float solid_background_tol_perc = 5 [default = .5]; - - // Force a border of this size in pixels on top and bottom. - optional int32 default_padding_px = 6 [default = 0]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc deleted file mode 100644 index d41013c3d..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator_test.cc +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/border_detection_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/benchmark.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -using mediapipe::Adopt; -using mediapipe::CalculatorGraphConfig; -using mediapipe::CalculatorRunner; -using mediapipe::ImageFormat; -using mediapipe::ImageFrame; -using mediapipe::Packet; -using mediapipe::PacketTypeSet; -using mediapipe::ParseTextProtoOrDie; -using mediapipe::Timestamp; -using mediapipe::autoflip::Border; - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfig[] = R"( - calculator: "BorderDetectionCalculator" - input_stream: "VIDEO:camera_frames" - output_stream: "DETECTED_BORDERS:regions" - options:{ - [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{ - border_object_padding_px: 0 - } - })"; - -const char kConfigPad[] = R"( - calculator: "BorderDetectionCalculator" - input_stream: "VIDEO:camera_frames" - output_stream: "DETECTED_BORDERS:regions" - options:{ - [mediapipe.autoflip.BorderDetectionCalculatorOptions.ext]:{ - default_padding_px: 10 - border_object_padding_px: 0 - } - })"; - -const int kTestFrameWidth = 640; -const int kTestFrameHeight = 480; - -const int kTestFrameLargeWidth = 1920; -const int kTestFrameLargeHeight = 1080; - -const int kTestFrameWidthTall = 1200; -const int kTestFrameHeightTall = 2001; - -TEST(BorderDetectionCalculatorTest, NoBorderTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(0, static_features.border().size()); - EXPECT_EQ(0, static_features.non_static_area().x()); - EXPECT_EQ(0, static_features.non_static_area().y()); - EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width()); - EXPECT_EQ(kTestFrameHeight, static_features.non_static_area().height()); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); -} - -TEST(BorderDetectionCalculatorTest, TopBorderTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - const int kTopBorderHeight = 50; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat sub_image = - input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight)); - sub_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(1, static_features.border().size()); - const auto& part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), 0); - EXPECT_EQ(part.border_position().width(), kTestFrameWidth); - EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(0, static_features.non_static_area().x()); - EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y()); - EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width()); - EXPECT_EQ(kTestFrameHeight - kTopBorderHeight + 1, - static_features.non_static_area().height()); -} - -TEST(BorderDetectionCalculatorTest, TopBorderPadTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigPad)); - - const int kTopBorderHeight = 50; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat sub_image = - input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight)); - sub_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(1, static_features.border().size()); - const auto& part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), 0); - EXPECT_EQ(part.border_position().width(), kTestFrameWidth); - EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(Border::TOP, part.relative_position()); - EXPECT_EQ(0, static_features.non_static_area().x()); - EXPECT_EQ(9 + kTopBorderHeight, static_features.non_static_area().y()); - EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width()); - EXPECT_EQ(kTestFrameHeight - 19 - kTopBorderHeight, - static_features.non_static_area().height()); -} - -TEST(BorderDetectionCalculatorTest, BottomBorderTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - const int kBottomBorderHeight = 50; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat bottom_image = - input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight, - kTestFrameWidth, kBottomBorderHeight)); - bottom_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(1, static_features.border().size()); - const auto& part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight); - EXPECT_EQ(part.border_position().width(), kTestFrameWidth); - EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(Border::BOTTOM, part.relative_position()); -} - -TEST(BorderDetectionCalculatorTest, TopBottomBorderTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - const int kBottomBorderHeight = 50; - const int kTopBorderHeight = 25; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat top_image = - input_mat(cv::Rect(0, 0, kTestFrameWidth, kTopBorderHeight)); - top_image.setTo(cv::Scalar(0, 255, 0)); - cv::Mat bottom_image = - input_mat(cv::Rect(0, kTestFrameHeight - kBottomBorderHeight, - kTestFrameWidth, kBottomBorderHeight)); - bottom_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(2, static_features.border().size()); - auto part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), 0); - EXPECT_EQ(part.border_position().width(), kTestFrameWidth); - EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(0, static_features.non_static_area().x()); - EXPECT_EQ(kTopBorderHeight - 1, static_features.non_static_area().y()); - EXPECT_EQ(kTestFrameWidth, static_features.non_static_area().width()); - EXPECT_EQ(kTestFrameHeight - kTopBorderHeight - kBottomBorderHeight + 2, - static_features.non_static_area().height()); - EXPECT_EQ(Border::TOP, part.relative_position()); - - part = static_features.border(1); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), kTestFrameHeight - kBottomBorderHeight); - EXPECT_EQ(part.border_position().width(), kTestFrameWidth); - EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2); - EXPECT_EQ(Border::BOTTOM, part.relative_position()); -} - -TEST(BorderDetectionCalculatorTest, TopBottomBorderTestAspect2) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - const int kBottomBorderHeight = 50; - const int kTopBorderHeight = 25; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidthTall, kTestFrameHeightTall); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat top_image = - input_mat(cv::Rect(0, 0, kTestFrameWidthTall, kTopBorderHeight)); - top_image.setTo(cv::Scalar(0, 255, 0)); - cv::Mat bottom_image = - input_mat(cv::Rect(0, kTestFrameHeightTall - kBottomBorderHeight, - kTestFrameWidthTall, kBottomBorderHeight)); - bottom_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(2, static_features.border().size()); - auto part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), 0); - EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall); - EXPECT_LT(std::abs(part.border_position().height() - kTopBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(Border::TOP, part.relative_position()); - - part = static_features.border(1); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), - kTestFrameHeightTall - kBottomBorderHeight); - EXPECT_EQ(part.border_position().width(), kTestFrameWidthTall); - EXPECT_LT(std::abs(part.border_position().height() - kBottomBorderHeight), 2); - EXPECT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(0, static_features.solid_background().b()); - EXPECT_EQ(Border::BOTTOM, part.relative_position()); -} - -TEST(BorderDetectionCalculatorTest, DominantColor) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfigPad); - node.mutable_options() - ->MutableExtension(BorderDetectionCalculatorOptions::ext) - ->set_solid_background_tol_perc(.25); - - auto runner = ::absl::make_unique(node); - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - - cv::Mat sub_image = input_mat(cv::Rect( - kTestFrameWidth / 2, 0, kTestFrameWidth / 2, kTestFrameHeight / 2)); - sub_image.setTo(cv::Scalar(0, 255, 0)); - - sub_image = input_mat(cv::Rect(0, kTestFrameHeight / 2, kTestFrameWidth / 2, - kTestFrameHeight / 2)); - sub_image.setTo(cv::Scalar(0, 0, 255)); - - sub_image = - input_mat(cv::Rect(0, 0, kTestFrameWidth / 2 + 50, kTestFrameHeight / 2)); - sub_image.setTo(cv::Scalar(255, 0, 0)); - - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("DETECTED_BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - ASSERT_EQ(0, static_features.border().size()); - ASSERT_TRUE(static_features.has_solid_background()); - EXPECT_EQ(0, static_features.solid_background().r()); - EXPECT_EQ(0, static_features.solid_background().g()); - EXPECT_EQ(255, static_features.solid_background().b()); -} - -void BM_Large(benchmark::State& state) { - for (auto _ : state) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfig)); - - const int kTopBorderHeight = 50; - - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameLargeWidth, kTestFrameLargeHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat sub_image = - input_mat(cv::Rect(0, 0, kTestFrameLargeWidth, kTopBorderHeight)); - sub_image.setTo(cv::Scalar(255, 0, 0)); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - } -} -BENCHMARK(BM_Large); - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc deleted file mode 100644 index e7c471abb..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc +++ /dev/null @@ -1,734 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -constexpr char kVideoFrame[] = "VIDEO"; -constexpr char kVideoSize[] = "VIDEO_SIZE"; -constexpr char kSalientRegions[] = "SALIENT_REGIONS"; -constexpr char kDetections[] = "DETECTIONS"; -constexpr char kDetectedBorders[] = "BORDERS"; -constexpr char kCropRect[] = "CROP_RECT"; -constexpr char kFirstCropRect[] = "FIRST_CROP_RECT"; -// Can be used to control whether an animated zoom should actually performed -// (configured through option us_to_first_rect). If provided, a non-zero integer -// will allow the animated zoom to be used when the first detections arrive. -constexpr char kAnimateZoom[] = "ANIMATE_ZOOM"; -// Can be used to control the maximum zoom; note that it is re-evaluated only -// upon change of input resolution. A value of 100 disables zooming and is the -// smallest allowed value. A value of 200 allows zooming such that a pixel of -// the input may cover up to four times its original area. Note that -// max_zoom_value_deg from options is always respected; MAX_ZOOM_PCT can only be -// used to limit zooming further. -constexpr char kMaxZoomFactorPercent[] = "MAX_ZOOM_FACTOR_PCT"; -// Field-of-view (degrees) of the camera's x-axis (width). -// TODO: Parameterize FOV based on camera specs. -constexpr float kFieldOfView = 60; -// A pointer to a ContentZoomingCalculatorStateCacheType in a side packet. -// Used to save state on Close and load state on Open in a new graph. -// Can be used to preserve state between graphs. -constexpr char kStateCache[] = "STATE_CACHE"; - -namespace mediapipe { -namespace autoflip { -using StateCacheType = ContentZoomingCalculatorStateCacheType; - -// Content zooming calculator zooms in on content when a detection has -// "only_required" set true or any raw detection input. It does this by -// computing the value of top/bottom borders to remove from the output and sends -// these to the SceneCroppingCalculator using BORDERS output or a full rect crop -// using CROP_RECT output. When more than one detections are received the -// zoom box is calculated as the union of the detections. Typical applications -// include mobile makeover and autofliplive face reframing. -class ContentZoomingCalculator : public CalculatorBase { - public: - ContentZoomingCalculator() : initialized_(false) {} - ~ContentZoomingCalculator() override {} - ContentZoomingCalculator(const ContentZoomingCalculator&) = delete; - ContentZoomingCalculator& operator=(const ContentZoomingCalculator&) = delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - absl::Status Close(mediapipe::CalculatorContext* cc) override; - - private: - // Tries to load state from a state-cache, if provided. Fallsback to - // initializing state if no cache or no value in the cache are available. - absl::Status MaybeLoadState(mediapipe::CalculatorContext* cc, int frame_width, - int frame_height); - // Saves state to a state-cache, if provided. - absl::Status SaveState(mediapipe::CalculatorContext* cc) const; - // Returns the factor for maximum zoom based on options and the - // kMaxZoomFactorPercent input (if present). - double GetMaxZoomFactor(mediapipe::CalculatorContext* cc) const; - // Initializes the calculator for the given frame size, creating path solvers - // and resetting history like last measured values. - absl::Status InitializeState(mediapipe::CalculatorContext* cc, - int frame_width, int frame_height); - // Adjusts state to work with an updated frame size. - absl::Status UpdateForResolutionChange(mediapipe::CalculatorContext* cc, - int frame_width, int frame_height); - // Returns true if we are animating to the first rect. - bool IsAnimatingToFirstRect(const Timestamp& timestamp) const; - // Builds the output rectangle when animating to the first rect. - absl::StatusOr GetAnimationRect( - int frame_width, int frame_height, const Timestamp& timestamp) const; - // Converts bounds to tilt offset, pan offset and height. - absl::Status ConvertToPanTiltZoom(float xmin, float xmax, float ymin, - float ymax, int* tilt_offset, - int* pan_offset, int* height); - // Sets max_frame_value_ and target_aspect_ - absl::Status UpdateAspectAndMax(); - ContentZoomingCalculatorOptions options_; - // Detection frame width/height. - int frame_height_; - int frame_width_; - // Path solver used to smooth top/bottom border crop values. - std::unique_ptr path_solver_zoom_; - std::unique_ptr path_solver_pan_; - std::unique_ptr path_solver_tilt_; - // Are parameters initialized. - bool initialized_; - // Stores the time of the first crop rectangle. This is used to control - // animating to it. Until a first crop rectangle was computed, it has - // the value Timestamp::Unset(). If animating is not requested, it receives - // the value Timestamp::Done() instead of the time. - Timestamp first_rect_timestamp_; - // Stores the first crop rectangle. - mediapipe::NormalizedRect first_rect_; - // Stores the time of the last "only_required" input. - int64 last_only_required_detection_; - // Rect values of last message with detection(s). - int last_measured_height_; - int last_measured_x_offset_; - int last_measured_y_offset_; - // Target aspect ratio. - float target_aspect_; - // Max size of bounding box. If input/output aspect ratios are the same, - // will be 1.0. Else, will be less than 1.0 to prevent exceeding the size of - // the image in either dimension. - float max_frame_value_; -}; -REGISTER_CALCULATOR(ContentZoomingCalculator); - -absl::Status ContentZoomingCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - RET_CHECK( - !(cc->Inputs().HasTag(kVideoFrame) && cc->Inputs().HasTag(kVideoSize))) - << "Provide only VIDEO or VIDEO_SIZE, not both."; - if (cc->Inputs().HasTag(kVideoFrame)) { - cc->Inputs().Tag(kVideoFrame).Set(); - } else if (cc->Inputs().HasTag(kVideoSize)) { - cc->Inputs().Tag(kVideoSize).Set>(); - } else { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Input VIDEO or VIDEO_SIZE must be provided."; - } - if (cc->Inputs().HasTag(kMaxZoomFactorPercent)) { - cc->Inputs().Tag(kMaxZoomFactorPercent).Set(); - } - if (cc->Inputs().HasTag(kSalientRegions)) { - cc->Inputs().Tag(kSalientRegions).Set(); - } - if (cc->Inputs().HasTag(kDetections)) { - cc->Inputs().Tag(kDetections).Set>(); - } - if (cc->Inputs().HasTag(kAnimateZoom)) { - cc->Inputs().Tag(kAnimateZoom).Set(); - } - if (cc->Outputs().HasTag(kDetectedBorders)) { - cc->Outputs().Tag(kDetectedBorders).Set(); - } - if (cc->Outputs().HasTag(kCropRect)) { - cc->Outputs().Tag(kCropRect).Set(); - } - if (cc->Outputs().HasTag(kFirstCropRect)) { - cc->Outputs().Tag(kFirstCropRect).Set(); - } - if (cc->InputSidePackets().HasTag(kStateCache)) { - cc->InputSidePackets().Tag(kStateCache).Set(); - } - return absl::OkStatus(); -} - -absl::Status ContentZoomingCalculator::Open(mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - if (options_.has_kinematic_options()) { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Deprecated kinematic_options was set, please set " - "kinematic_options_zoom and kinematic_options_tilt."; - } - if (options_.has_min_motion_to_reframe()) { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Deprecated min_motion_to_reframe was set, please set " - "in kinematic_options_zoom and kinematic_options_tilt " - "directly."; - } - return absl::OkStatus(); -} - -absl::Status ContentZoomingCalculator::Close(mediapipe::CalculatorContext* cc) { - if (initialized_) { - MP_RETURN_IF_ERROR(SaveState(cc)); - } - return absl::OkStatus(); -} - -absl::Status ContentZoomingCalculator::ConvertToPanTiltZoom( - float xmin, float xmax, float ymin, float ymax, int* tilt_offset, - int* pan_offset, int* height) { - // Find center of the y-axis offset (for tilt control). - float y_center = ymin + (ymax - ymin) / 2; - // Find center of the x-axis offset (for pan control). - float x_center = xmin + (xmax - xmin) / 2; - // Find size and apply scale factor to y-axis. - float fit_size_raw = - fmax((ymax - ymin) / options_.scale_factor(), xmax - xmin); - // Apply max frame for cases where the target size is different than input - // frame size. - float fit_size = fmin(max_frame_value_, fit_size_raw); - // Prevent box from extending beyond the image. - if (y_center - fit_size / 2 < 0) { - y_center = fit_size / 2; - } else if (y_center + fit_size / 2 > 1) { - y_center = 1 - fit_size / 2; - } - if (x_center - fit_size / 2 < 0) { - x_center = fit_size / 2; - } else if (x_center + fit_size / 2 > 1) { - x_center = 1 - fit_size / 2; - } - // Scale to pixel coordinates. - *tilt_offset = frame_height_ * y_center; - *pan_offset = frame_width_ * x_center; - *height = frame_height_ * fit_size_raw; - return absl::OkStatus(); -} - -namespace { -mediapipe::LocationData::RelativeBoundingBox ShiftDetection( - const mediapipe::LocationData::RelativeBoundingBox& relative_bounding_box, - const float y_offset_percent, const float x_offset_percent) { - auto shifted_bb = relative_bounding_box; - shifted_bb.set_ymin(relative_bounding_box.ymin() + - relative_bounding_box.height() * y_offset_percent); - shifted_bb.set_xmin(relative_bounding_box.xmin() + - relative_bounding_box.width() * x_offset_percent); - return shifted_bb; -} -mediapipe::autoflip::RectF ShiftDetection( - const mediapipe::autoflip::RectF& relative_bounding_box, - const float y_offset_percent, const float x_offset_percent) { - auto shifted_bb = relative_bounding_box; - shifted_bb.set_y(relative_bounding_box.y() + - relative_bounding_box.height() * y_offset_percent); - shifted_bb.set_x(relative_bounding_box.x() + - relative_bounding_box.width() * x_offset_percent); - return shifted_bb; -} -absl::Status UpdateRanges(const SalientRegion& region, - const float shift_vertical, - const float shift_horizontal, float* xmin, - float* xmax, float* ymin, float* ymax) { - if (!region.has_location_normalized()) { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "SalientRegion did not have location normalized set."; - } - auto location = ShiftDetection(region.location_normalized(), shift_vertical, - shift_horizontal); - *xmin = fmin(*xmin, location.x()); - *xmax = fmax(*xmax, location.x() + location.width()); - *ymin = fmin(*ymin, location.y()); - *ymax = fmax(*ymax, location.y() + location.height()); - - return absl::OkStatus(); -} -absl::Status UpdateRanges(const mediapipe::Detection& detection, - const float shift_vertical, - const float shift_horizontal, float* xmin, - float* xmax, float* ymin, float* ymax) { - RET_CHECK(detection.location_data().format() == - mediapipe::LocationData::RELATIVE_BOUNDING_BOX) - << "Face detection input is lacking required relative_bounding_box()"; - const auto& location = - ShiftDetection(detection.location_data().relative_bounding_box(), - shift_vertical, shift_horizontal); - *xmin = fmin(*xmin, location.xmin()); - *xmax = fmax(*xmax, location.xmin() + location.width()); - *ymin = fmin(*ymin, location.ymin()); - *ymax = fmax(*ymax, location.ymin() + location.height()); - - return absl::OkStatus(); -} -void MakeStaticFeatures(const int top_border, const int bottom_border, - const int frame_width, const int frame_height, - StaticFeatures* static_feature) { - auto border_top = static_feature->add_border(); - border_top->set_relative_position(Border::TOP); - border_top->mutable_border_position()->set_x(0); - border_top->mutable_border_position()->set_y(0); - border_top->mutable_border_position()->set_width(frame_width); - border_top->mutable_border_position()->set_height(top_border); - - auto border_bottom = static_feature->add_border(); - border_bottom->set_relative_position(Border::BOTTOM); - border_bottom->mutable_border_position()->set_x(0); - border_bottom->mutable_border_position()->set_y(frame_height - bottom_border); - border_bottom->mutable_border_position()->set_width(frame_width); - border_bottom->mutable_border_position()->set_height(bottom_border); -} -absl::Status GetVideoResolution(mediapipe::CalculatorContext* cc, - int* frame_width, int* frame_height) { - if (cc->Inputs().HasTag(kVideoFrame)) { - *frame_width = cc->Inputs().Tag(kVideoFrame).Get().Width(); - *frame_height = cc->Inputs().Tag(kVideoFrame).Get().Height(); - } else if (cc->Inputs().HasTag(kVideoSize)) { - *frame_width = - cc->Inputs().Tag(kVideoSize).Get>().first; - *frame_height = - cc->Inputs().Tag(kVideoSize).Get>().second; - } else { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Input VIDEO or VIDEO_SIZE must be provided."; - } - return absl::OkStatus(); -} -} // namespace - -absl::Status ContentZoomingCalculator::UpdateAspectAndMax() { - max_frame_value_ = 1.0; - target_aspect_ = frame_width_ / static_cast(frame_height_); - // If target size is set and wider than input aspect, make sure to always - // crop the min required amount. - if (options_.has_target_size()) { - RET_CHECK_GT(options_.target_size().width(), 0) - << "Provided target width not valid."; - RET_CHECK_GT(options_.target_size().height(), 0) - << "Provided target height not valid."; - float input_aspect = frame_width_ / static_cast(frame_height_); - target_aspect_ = options_.target_size().width() / - static_cast(options_.target_size().height()); - max_frame_value_ = - std::min(input_aspect / target_aspect_, target_aspect_ / input_aspect); - } - return absl::OkStatus(); -} - -absl::Status ContentZoomingCalculator::MaybeLoadState( - mediapipe::CalculatorContext* cc, int frame_width, int frame_height) { - const auto* state_cache = - cc->InputSidePackets().HasTag(kStateCache) - ? cc->InputSidePackets().Tag(kStateCache).Get() - : nullptr; - if (!state_cache || !state_cache->has_value()) { - return InitializeState(cc, frame_width, frame_height); - } - - const ContentZoomingCalculatorState& state = state_cache->value(); - frame_width_ = state.frame_width; - frame_height_ = state.frame_height; - path_solver_pan_ = - std::make_unique(state.path_solver_pan); - path_solver_tilt_ = - std::make_unique(state.path_solver_tilt); - path_solver_zoom_ = - std::make_unique(state.path_solver_zoom); - first_rect_timestamp_ = state.first_rect_timestamp; - first_rect_ = state.first_rect; - last_only_required_detection_ = state.last_only_required_detection; - last_measured_height_ = state.last_measured_height; - last_measured_x_offset_ = state.last_measured_x_offset; - last_measured_y_offset_ = state.last_measured_y_offset; - MP_RETURN_IF_ERROR(UpdateAspectAndMax()); - - return UpdateForResolutionChange(cc, frame_width, frame_height); -} - -absl::Status ContentZoomingCalculator::SaveState( - mediapipe::CalculatorContext* cc) const { - auto* state_cache = - cc->InputSidePackets().HasTag(kStateCache) - ? cc->InputSidePackets().Tag(kStateCache).Get() - : nullptr; - if (!state_cache) { - return absl::OkStatus(); - } - - *state_cache = ContentZoomingCalculatorState{ - .frame_height = frame_height_, - .frame_width = frame_width_, - .path_solver_zoom = *path_solver_zoom_, - .path_solver_pan = *path_solver_pan_, - .path_solver_tilt = *path_solver_tilt_, - .first_rect_timestamp = first_rect_timestamp_, - .first_rect = first_rect_, - .last_only_required_detection = last_only_required_detection_, - .last_measured_height = last_measured_height_, - .last_measured_x_offset = last_measured_x_offset_, - .last_measured_y_offset = last_measured_y_offset_, - }; - return absl::OkStatus(); -} - -double ContentZoomingCalculator::GetMaxZoomFactor( - mediapipe::CalculatorContext* cc) const { - double max_zoom_value = - options_.max_zoom_value_deg() / static_cast(kFieldOfView); - if (cc->Inputs().HasTag(kMaxZoomFactorPercent)) { - const double factor = std::max( - 1.0, cc->Inputs().Tag(kMaxZoomFactorPercent).Get() / 100.0); - max_zoom_value = std::max(max_zoom_value, 1.0 / factor); - } - return max_zoom_value; -} - -absl::Status ContentZoomingCalculator::InitializeState( - mediapipe::CalculatorContext* cc, int frame_width, int frame_height) { - frame_width_ = frame_width; - frame_height_ = frame_height; - path_solver_pan_ = std::make_unique( - options_.kinematic_options_pan(), 0, frame_width_, - static_cast(frame_width_) / kFieldOfView); - path_solver_tilt_ = std::make_unique( - options_.kinematic_options_tilt(), 0, frame_height_, - static_cast(frame_height_) / kFieldOfView); - MP_RETURN_IF_ERROR(UpdateAspectAndMax()); - int min_zoom_size = frame_height_ * GetMaxZoomFactor(cc); - path_solver_zoom_ = std::make_unique( - options_.kinematic_options_zoom(), min_zoom_size, - max_frame_value_ * frame_height_, - static_cast(frame_height_) / kFieldOfView); - first_rect_timestamp_ = Timestamp::Unset(); - last_only_required_detection_ = 0; - last_measured_height_ = max_frame_value_ * frame_height_; - last_measured_x_offset_ = target_aspect_ * frame_width_; - last_measured_y_offset_ = frame_width_ / 2; - return absl::OkStatus(); -} - -absl::Status ContentZoomingCalculator::UpdateForResolutionChange( - mediapipe::CalculatorContext* cc, int frame_width, int frame_height) { - // Update state for change in input resolution. - if (frame_width_ != frame_width || frame_height_ != frame_height) { - double width_scale = frame_width / static_cast(frame_width_); - double height_scale = frame_height / static_cast(frame_height_); - last_measured_height_ = last_measured_height_ * height_scale; - last_measured_y_offset_ = last_measured_y_offset_ * height_scale; - last_measured_x_offset_ = last_measured_x_offset_ * width_scale; - frame_width_ = frame_width; - frame_height_ = frame_height; - MP_RETURN_IF_ERROR(UpdateAspectAndMax()); - MP_RETURN_IF_ERROR(path_solver_pan_->UpdateMinMaxLocation(0, frame_width_)); - MP_RETURN_IF_ERROR( - path_solver_tilt_->UpdateMinMaxLocation(0, frame_height_)); - int min_zoom_size = frame_height_ * GetMaxZoomFactor(cc); - MP_RETURN_IF_ERROR(path_solver_zoom_->UpdateMinMaxLocation( - min_zoom_size, max_frame_value_ * frame_height_)); - MP_RETURN_IF_ERROR(path_solver_zoom_->UpdatePixelsPerDegree( - static_cast(frame_height_) / kFieldOfView)); - } - return absl::OkStatus(); -} - -bool ContentZoomingCalculator::IsAnimatingToFirstRect( - const Timestamp& timestamp) const { - if (options_.us_to_first_rect() == 0 || - first_rect_timestamp_ == Timestamp::Unset() || - first_rect_timestamp_ == Timestamp::Done()) { - return false; - } - - const int64 delta_us = (timestamp - first_rect_timestamp_).Value(); - return (0 <= delta_us && delta_us <= options_.us_to_first_rect()); -} - -namespace { -double easeInQuad(double t) { return t * t; } -double easeOutQuad(double t) { return -1 * t * (t - 2); } -double easeInOutQuad(double t) { - if (t < 0.5) { - return easeInQuad(t * 2) * 0.5; - } else { - return easeOutQuad(t * 2 - 1) * 0.5 + 0.5; - } -} -double lerp(double a, double b, double i) { return a * (1 - i) + b * i; } -} // namespace - -absl::StatusOr ContentZoomingCalculator::GetAnimationRect( - int frame_width, int frame_height, const Timestamp& timestamp) const { - RET_CHECK(IsAnimatingToFirstRect(timestamp)) - << "Must only be called if animating to first rect."; - - const int64 delta_us = (timestamp - first_rect_timestamp_).Value(); - const int64 delay = options_.us_to_first_rect_delay(); - const double interpolation = easeInOutQuad(std::max( - 0.0, (delta_us - delay) / - static_cast(options_.us_to_first_rect() - delay))); - - const double x_center = lerp(0.5, first_rect_.x_center(), interpolation); - const double y_center = lerp(0.5, first_rect_.y_center(), interpolation); - const double width = lerp(1.0, first_rect_.width(), interpolation); - const double height = lerp(1.0, first_rect_.height(), interpolation); - - mediapipe::Rect gpu_rect; - gpu_rect.set_x_center(x_center * frame_width); - gpu_rect.set_width(width * frame_width); - gpu_rect.set_y_center(y_center * frame_height); - gpu_rect.set_height(height * frame_height); - return gpu_rect; -} - -absl::Status ContentZoomingCalculator::Process( - mediapipe::CalculatorContext* cc) { - // For async subgraph support, return on empty video size packets. - if (cc->Inputs().HasTag(kVideoSize) && - cc->Inputs().Tag(kVideoSize).IsEmpty()) { - return absl::OkStatus(); - } - int frame_width, frame_height; - MP_RETURN_IF_ERROR(GetVideoResolution(cc, &frame_width, &frame_height)); - - // Init on first call or re-init always if configured to be stateless. - if (!initialized_) { - MP_RETURN_IF_ERROR(MaybeLoadState(cc, frame_width, frame_height)); - initialized_ = !options_.is_stateless(); - } else { - MP_RETURN_IF_ERROR( - UpdateForResolutionChange(cc, frame_width, frame_height)); - } - - bool only_required_found = false; - - // Compute the box that contains all "is_required" detections. - float xmin = 1, ymin = 1, xmax = 0, ymax = 0; - if (cc->Inputs().HasTag(kSalientRegions)) { - auto detection_set = cc->Inputs().Tag(kSalientRegions).Get(); - for (const auto& region : detection_set.detections()) { - if (!region.only_required()) { - continue; - } - only_required_found = true; - MP_RETURN_IF_ERROR(UpdateRanges( - region, options_.detection_shift_vertical(), - options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax)); - } - } - - if (cc->Inputs().HasTag(kDetections)) { - if (cc->Inputs().Tag(kDetections).IsEmpty()) { - if (last_only_required_detection_ == 0) { - // If no detections are available and we never had any, - // simply return the full-image rectangle as crop-rect. - if (cc->Outputs().HasTag(kCropRect)) { - auto default_rect = absl::make_unique(); - default_rect->set_x_center(frame_width_ / 2); - default_rect->set_y_center(frame_height_ / 2); - default_rect->set_width(frame_width_); - default_rect->set_height(frame_height_); - cc->Outputs().Tag(kCropRect).Add(default_rect.release(), - Timestamp(cc->InputTimestamp())); - } - // Also provide a first crop rect: in this case a zero-sized one. - if (cc->Outputs().HasTag(kFirstCropRect)) { - cc->Outputs() - .Tag(kFirstCropRect) - .Add(new mediapipe::NormalizedRect(), - Timestamp(cc->InputTimestamp())); - } - return absl::OkStatus(); - } - } else { - auto raw_detections = cc->Inputs() - .Tag(kDetections) - .Get>(); - for (const auto& detection : raw_detections) { - only_required_found = true; - MP_RETURN_IF_ERROR(UpdateRanges( - detection, options_.detection_shift_vertical(), - options_.detection_shift_horizontal(), &xmin, &xmax, &ymin, &ymax)); - } - } - } - - const bool may_start_animation = (options_.us_to_first_rect() != 0) && - (!cc->Inputs().HasTag(kAnimateZoom) || - cc->Inputs().Tag(kAnimateZoom).Get()); - bool is_animating = IsAnimatingToFirstRect(cc->InputTimestamp()); - - int offset_y, height, offset_x; - if (!is_animating && options_.start_zoomed_out() && !may_start_animation && - first_rect_timestamp_ == Timestamp::Unset()) { - // If we should start zoomed out and won't be doing an animation, - // initialize the path solvers using the full frame, ignoring detections. - height = max_frame_value_ * frame_height_; - offset_x = (target_aspect_ * height) / 2; - offset_y = frame_height_ / 2; - } else if (!is_animating && only_required_found) { - // Convert bounds to tilt/zoom and in pixel coordinates. - MP_RETURN_IF_ERROR(ConvertToPanTiltZoom(xmin, xmax, ymin, ymax, &offset_y, - &offset_x, &height)); - // A only required detection was found. - last_only_required_detection_ = cc->InputTimestamp().Microseconds(); - last_measured_height_ = height; - last_measured_x_offset_ = offset_x; - last_measured_y_offset_ = offset_y; - } else if (!is_animating && cc->InputTimestamp().Microseconds() - - last_only_required_detection_ >= - options_.us_before_zoomout()) { - // No only_require detections found within salient regions packets - // arriving since us_before_zoomout duration. - height = max_frame_value_ * frame_height_ + - (options_.kinematic_options_zoom().min_motion_to_reframe() * - (static_cast(frame_height_) / kFieldOfView)); - offset_x = (target_aspect_ * height) / 2; - offset_y = frame_height_ / 2; - } else { - // Either animating to the first rectangle, or - // no only detection found but using last detection due to - // duration_before_zoomout_us setting. - height = last_measured_height_; - offset_x = last_measured_x_offset_; - offset_y = last_measured_y_offset_; - } - - // Check if the camera is changing in pan, tilt or zoom. If the camera is in - // motion disable temporal filtering. - bool pan_state, tilt_state, zoom_state; - MP_RETURN_IF_ERROR(path_solver_pan_->PredictMotionState( - offset_x, cc->InputTimestamp().Microseconds(), &pan_state)); - MP_RETURN_IF_ERROR(path_solver_tilt_->PredictMotionState( - offset_y, cc->InputTimestamp().Microseconds(), &tilt_state)); - MP_RETURN_IF_ERROR(path_solver_zoom_->PredictMotionState( - height, cc->InputTimestamp().Microseconds(), &zoom_state)); - if (pan_state || tilt_state || zoom_state) { - path_solver_pan_->ClearHistory(); - path_solver_tilt_->ClearHistory(); - path_solver_zoom_->ClearHistory(); - } - - // Compute smoothed zoom camera path. - MP_RETURN_IF_ERROR(path_solver_zoom_->AddObservation( - height, cc->InputTimestamp().Microseconds())); - int path_height; - MP_RETURN_IF_ERROR(path_solver_zoom_->GetState(&path_height)); - int path_width = path_height * target_aspect_; - - // Update pixel-per-degree value for pan/tilt. - int target_height; - MP_RETURN_IF_ERROR(path_solver_zoom_->GetTargetPosition(&target_height)); - int target_width = target_height * target_aspect_; - MP_RETURN_IF_ERROR(path_solver_pan_->UpdatePixelsPerDegree( - static_cast(target_width) / kFieldOfView)); - MP_RETURN_IF_ERROR(path_solver_tilt_->UpdatePixelsPerDegree( - static_cast(target_height) / kFieldOfView)); - - // Compute smoothed pan/tilt paths. - MP_RETURN_IF_ERROR(path_solver_pan_->AddObservation( - offset_x, cc->InputTimestamp().Microseconds())); - MP_RETURN_IF_ERROR(path_solver_tilt_->AddObservation( - offset_y, cc->InputTimestamp().Microseconds())); - int path_offset_x; - MP_RETURN_IF_ERROR(path_solver_pan_->GetState(&path_offset_x)); - int path_offset_y; - MP_RETURN_IF_ERROR(path_solver_tilt_->GetState(&path_offset_y)); - - // Prevent box from extending beyond the image after camera smoothing. - if (path_offset_y - ceil(path_height / 2.0) < 0) { - path_offset_y = ceil(path_height / 2.0); - } else if (path_offset_y + ceil(path_height / 2.0) > frame_height_) { - path_offset_y = frame_height_ - ceil(path_height / 2.0); - } - - if (path_offset_x - ceil(path_width / 2.0) < 0) { - path_offset_x = ceil(path_width / 2.0); - } else if (path_offset_x + ceil(path_width / 2.0) > frame_width_) { - path_offset_x = frame_width_ - ceil(path_width / 2.0); - } - - // Convert to top/bottom borders to remove. - int path_top = path_offset_y - path_height / 2; - int path_bottom = frame_height_ - (path_offset_y + path_height / 2); - - // Transmit result downstream to scenecroppingcalculator. - if (cc->Outputs().HasTag(kDetectedBorders)) { - std::unique_ptr features = - absl::make_unique(); - MakeStaticFeatures(path_top, path_bottom, frame_width_, frame_height_, - features.get()); - cc->Outputs() - .Tag(kDetectedBorders) - .AddPacket(Adopt(features.release()).At(cc->InputTimestamp())); - } - - // Record the first crop rectangle - if (first_rect_timestamp_ == Timestamp::Unset()) { - first_rect_.set_x_center(path_offset_x / static_cast(frame_width_)); - first_rect_.set_width(path_height * target_aspect_ / - static_cast(frame_width_)); - first_rect_.set_y_center(path_offset_y / static_cast(frame_height_)); - first_rect_.set_height(path_height / static_cast(frame_height_)); - - // Record the time to serve as departure point for the animation. - // If we are not allowed to start the animation, set Timestamp::Done. - first_rect_timestamp_ = - may_start_animation ? cc->InputTimestamp() : Timestamp::Done(); - // After setting the first rectangle, check whether we should animate to it. - is_animating = IsAnimatingToFirstRect(cc->InputTimestamp()); - } - - // Transmit downstream to glcroppingcalculator. - if (cc->Outputs().HasTag(kCropRect)) { - std::unique_ptr gpu_rect; - if (is_animating) { - auto rect = - GetAnimationRect(frame_width, frame_height, cc->InputTimestamp()); - MP_RETURN_IF_ERROR(rect.status()); - gpu_rect = absl::make_unique(*rect); - } else { - gpu_rect = absl::make_unique(); - gpu_rect->set_x_center(path_offset_x); - gpu_rect->set_width(path_height * target_aspect_); - gpu_rect->set_y_center(path_offset_y); - gpu_rect->set_height(path_height); - } - cc->Outputs().Tag(kCropRect).Add(gpu_rect.release(), - Timestamp(cc->InputTimestamp())); - } - - if (cc->Outputs().HasTag(kFirstCropRect)) { - cc->Outputs() - .Tag(kFirstCropRect) - .Add(new mediapipe::NormalizedRect(first_rect_), - Timestamp(cc->InputTimestamp())); - } - - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto deleted file mode 100644 index 6516ed21f..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto"; -import "mediapipe/framework/calculator.proto"; - -// NextTag: 18 -message ContentZoomingCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional ContentZoomingCalculatorOptions ext = 313091992; - } - // Amount the only required area should fill the image. When set to 1, - // reframing is done to remove content to the very edge of the salient region - // bounding box. When a smaller value is used, zooming will be done to fill - // this ratio of the frame. - optional float scale_factor = 1 [default = .9]; - // Kinematic options for zooming. - optional KinematicOptions kinematic_options_zoom = 6; - // Kinematic options for tilt (y-axis reframing.) - optional KinematicOptions kinematic_options_tilt = 7; - // Kinematic options for pan (x-axis reframing.) - optional KinematicOptions kinematic_options_pan = 10; - // Duration (in MicroSeconds) before returning to fully zoomed out position - // when no "only_required" frames are received. - optional int64 us_before_zoomout = 9 [default = 1000000]; - // Value of target output size, required to be set if different than input. - // Should match target_width and target_height in croppingcalculator. - message Size { - optional int64 width = 1; - optional int64 height = 2; - } - optional Size target_size = 8; - // Amount to shift an input detection as a ratio of the size (positive: - // down/right, negative: up/left). Use a negative value to increase padding - // above/left of an object, positive to increase padding below/right of an - // object. - optional float detection_shift_vertical = 11 [default = 0.0]; - optional float detection_shift_horizontal = 12 [default = 0.0]; - - // Defines the smallest value in degrees the camera is permitted to zoom. - optional float max_zoom_value_deg = 13 [default = 35]; - - // Whether to keep state between frames or to compute the final crop rect. - optional bool is_stateless = 14 [default = false]; - - // If true, on the first packet start with the camera zoomed out and then zoom - // in on the subject. If false, the camera will start zoomed in on the - // subject. - optional bool start_zoomed_out = 17 [default = false]; - - // Duration (in MicroSeconds) for animating to the first crop rect. - // Note that if set, takes precedence over start_zoomed_out. - optional int64 us_to_first_rect = 15 [default = 0]; - // Duration (in MicroSeconds) to delay animating to the first crop rect. - // Used only if us_to_first_rect is set and is interpreted as part of the - // us_to_first_rect time budget. - optional int64 us_to_first_rect_delay = 16 [default = 0]; - - // Deprecated parameters - optional KinematicOptions kinematic_options = 2 [deprecated = true]; - optional int64 min_motion_to_reframe = 4 [deprecated = true]; - optional float min_vertical_zoom = 5 [default = 1, deprecated = true]; - optional int64 frames_before_zoomout = 3 [default = 30, deprecated = true]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h deleted file mode 100644 index c01a6fbb5..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { -namespace autoflip { - -struct ContentZoomingCalculatorState { - int frame_height = -1; - int frame_width = -1; - // Path solver used to smooth top/bottom border crop values. - KinematicPathSolver path_solver_zoom; - KinematicPathSolver path_solver_pan; - KinematicPathSolver path_solver_tilt; - // Stores the time of the first crop rectangle. - Timestamp first_rect_timestamp; - // Stores the first crop rectangle. - mediapipe::NormalizedRect first_rect; - // Stores the time of the last "only_required" input. - int64 last_only_required_detection = 0; - // Rect values of last message with detection(s). - int last_measured_height = 0; - int last_measured_x_offset = 0; - int last_measured_y_offset = 0; -}; - -using ContentZoomingCalculatorStateCacheType = - std::optional; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_CONTENT_ZOOMING_CALCULATOR_STATE_H_ diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc deleted file mode 100644 index 32ee84efa..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc +++ /dev/null @@ -1,977 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_state.h" -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/formats/rect.pb.h" -#include "mediapipe/framework/port/benchmark.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfigA[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO:camera_frames" - input_stream: "SALIENT_REGIONS:detection_set" - output_stream: "BORDERS:borders" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -const char kConfigB[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO:camera_frames" - input_stream: "SALIENT_REGIONS:detection_set" - output_stream: "BORDERS:borders" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - target_size { - width: 1000 - height: 500 - } - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -const char kConfigC[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO_SIZE:size" - input_stream: "SALIENT_REGIONS:detection_set" - output_stream: "BORDERS:borders" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -const char kConfigD[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO_SIZE:size" - input_stream: "DETECTIONS:detections" - output_stream: "CROP_RECT:rect" - output_stream: "FIRST_CROP_RECT:first_rect" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -const char kConfigE[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO_SIZE:size" - input_stream: "DETECTIONS:detections" - input_stream: "ANIMATE_ZOOM:animate_zoom" - output_stream: "CROP_RECT:rect" - output_stream: "FIRST_CROP_RECT:first_rect" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -const char kConfigF[] = R"( - calculator: "ContentZoomingCalculator" - input_stream: "VIDEO_SIZE:size" - input_stream: "DETECTIONS:detections" - input_stream: "MAX_ZOOM_FACTOR_PCT:max_zoom_factor_pct" - output_stream: "CROP_RECT:rect" - output_stream: "FIRST_CROP_RECT:first_rect" - options: { - [mediapipe.autoflip.ContentZoomingCalculatorOptions.ext]: { - max_zoom_value_deg: 0 - kinematic_options_zoom { - min_motion_to_reframe: 1.2 - } - kinematic_options_tilt { - min_motion_to_reframe: 1.2 - } - kinematic_options_pan { - min_motion_to_reframe: 1.2 - } - } - } - )"; - -void CheckBorder(const StaticFeatures& static_features, int width, int height, - int top_border, int bottom_border) { - ASSERT_EQ(2, static_features.border().size()); - auto part = static_features.border(0); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), 0); - EXPECT_EQ(part.border_position().width(), width); - EXPECT_EQ(part.border_position().height(), top_border); - EXPECT_EQ(Border::TOP, part.relative_position()); - - part = static_features.border(1); - EXPECT_EQ(part.border_position().x(), 0); - EXPECT_EQ(part.border_position().y(), height - bottom_border); - EXPECT_EQ(part.border_position().width(), width); - EXPECT_EQ(part.border_position().height(), bottom_border); - EXPECT_EQ(Border::BOTTOM, part.relative_position()); -} - -struct AddDetectionFlags { - std::optional animated_zoom; - std::optional max_zoom_factor_percent; -}; - -void AddDetectionFrameSize(const cv::Rect_& position, const int64 time, - const int width, const int height, - CalculatorRunner* runner, - const AddDetectionFlags& flags = {}) { - auto detections = std::make_unique>(); - if (position.width > 0 && position.height > 0) { - mediapipe::Detection detection; - detection.mutable_location_data()->set_format( - mediapipe::LocationData::RELATIVE_BOUNDING_BOX); - detection.mutable_location_data() - ->mutable_relative_bounding_box() - ->set_height(position.height); - detection.mutable_location_data() - ->mutable_relative_bounding_box() - ->set_width(position.width); - detection.mutable_location_data() - ->mutable_relative_bounding_box() - ->set_xmin(position.x); - detection.mutable_location_data() - ->mutable_relative_bounding_box() - ->set_ymin(position.y); - detections->push_back(detection); - } - runner->MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back(Adopt(detections.release()).At(Timestamp(time))); - - auto input_size = ::absl::make_unique>(width, height); - runner->MutableInputs() - ->Tag("VIDEO_SIZE") - .packets.push_back(Adopt(input_size.release()).At(Timestamp(time))); - - if (flags.animated_zoom.has_value()) { - runner->MutableInputs() - ->Tag("ANIMATE_ZOOM") - .packets.push_back( - mediapipe::MakePacket(flags.animated_zoom.value()) - .At(Timestamp(time))); - } - - if (flags.max_zoom_factor_percent.has_value()) { - runner->MutableInputs() - ->Tag("MAX_ZOOM_FACTOR_PCT") - .packets.push_back( - mediapipe::MakePacket(flags.max_zoom_factor_percent.value()) - .At(Timestamp(time))); - } -} - -void AddDetection(const cv::Rect_& position, const int64 time, - CalculatorRunner* runner) { - AddDetectionFrameSize(position, time, 1000, 1000, runner); -} - -void CheckCropRect(const int x_center, const int y_center, const int width, - const int height, const int frame_number, - const std::vector& output_packets) { - ASSERT_GT(output_packets.size(), frame_number); - const auto& rect = output_packets[frame_number].Get(); - EXPECT_EQ(rect.x_center(), x_center); - EXPECT_EQ(rect.y_center(), y_center); - EXPECT_EQ(rect.width(), width); - EXPECT_EQ(rect.height(), height); -} -TEST(ContentZoomingCalculatorTest, ZoomTest) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigA)); - auto detection_set = std::make_unique(); - auto* detection = detection_set->add_detections(); - detection->set_only_required(true); - auto* location = detection->mutable_location_normalized(); - location->set_height(.1); - location->set_width(.1); - location->set_x(.4); - location->set_y(.5); - - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, 1000, 1000); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp(0))); - - runner->MutableInputs() - ->Tag("SALIENT_REGIONS") - .packets.push_back(Adopt(detection_set.release()).At(Timestamp(0))); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - CheckBorder(static_features, 1000, 1000, 495, 395); -} - -#if 0 -TEST(ContentZoomingCalculatorTest, ZoomTestFullPTZ) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigD)); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, PanConfig) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0); - options->mutable_kinematic_options_pan()->set_update_rate_seconds(2); - options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - AddDetection(cv::Rect_(.45, .55, .15, .15), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(483, 550, 111, 111, 1, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, PanConfigWithCache) { - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache; - auto config = ParseTextProtoOrDie(kConfigD); - config.add_input_side_packet("STATE_CACHE:state_cache"); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(0.0); - options->mutable_kinematic_options_pan()->set_update_rate_seconds(2); - options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0); - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetection(cv::Rect_(.45, .55, .15, .15), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(483, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - // Now repeat the last frame for a new runner without the cache to see a reset - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr); - AddDetection(cv::Rect_(.45, .55, .15, .15), 2000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost. - runner->Outputs().Tag("CROP_RECT").packets); - } -} - -TEST(ContentZoomingCalculatorTest, TiltConfig) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(0.0); - options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(50.0); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - AddDetection(cv::Rect_(.45, .55, .15, .15), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(450, 583, 111, 111, 1, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ZoomConfig) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0); - options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - AddDetection(cv::Rect_(.45, .55, .15, .15), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(450, 550, 139, 139, 1, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ZoomConfigWithCache) { - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache; - auto config = ParseTextProtoOrDie(kConfigD); - config.add_input_side_packet("STATE_CACHE:state_cache"); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_tilt()->set_min_motion_to_reframe(50.0); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(0.0); - options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2); - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetection(cv::Rect_(.4, .5, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetection(cv::Rect_(.45, .55, .15, .15), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(450, 550, 139, 139, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - // Now repeat the last frame for a new runner without the cache to see a reset - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(nullptr); - AddDetection(cv::Rect_(.45, .55, .15, .15), 2000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(525, 625, 166, 166, 0, // Without a cache, state was lost. - runner->Outputs().Tag("CROP_RECT").packets); - } -} - -TEST(ContentZoomingCalculatorTest, MinAspectBorderValues) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigB)); - auto detection_set = std::make_unique(); - auto* detection = detection_set->add_detections(); - detection->set_only_required(true); - auto* location = detection->mutable_location_normalized(); - location->set_height(1); - location->set_width(1); - location->set_x(0); - location->set_y(0); - - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, 1000, 1000); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp(0))); - - runner->MutableInputs() - ->Tag("SALIENT_REGIONS") - .packets.push_back(Adopt(detection_set.release()).At(Timestamp(0))); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - CheckBorder(static_features, 1000, 1000, 250, 250); -} - -TEST(ContentZoomingCalculatorTest, TwoFacesWide) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigA)); - auto detection_set = std::make_unique(); - auto* detection = detection_set->add_detections(); - detection->set_only_required(true); - auto* location = detection->mutable_location_normalized(); - location->set_height(.2); - location->set_width(.2); - location->set_x(.2); - location->set_y(.4); - - location = detection->mutable_location_normalized(); - location->set_height(.2); - location->set_width(.2); - location->set_x(.6); - location->set_y(.4); - - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, 1000, 1000); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp(0))); - - runner->MutableInputs() - ->Tag("SALIENT_REGIONS") - .packets.push_back(Adopt(detection_set.release()).At(Timestamp(0))); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - - CheckBorder(static_features, 1000, 1000, 389, 389); -} - -TEST(ContentZoomingCalculatorTest, NoDetectionOnInit) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigA)); - auto detection_set = std::make_unique(); - - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, 1000, 1000); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp(0))); - - runner->MutableInputs() - ->Tag("SALIENT_REGIONS") - .packets.push_back(Adopt(detection_set.release()).At(Timestamp(0))); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - - CheckBorder(static_features, 1000, 1000, 0, 0); -} - -TEST(ContentZoomingCalculatorTest, ZoomTestPairSize) { - auto runner = ::absl::make_unique( - ParseTextProtoOrDie(kConfigC)); - auto detection_set = std::make_unique(); - auto* detection = detection_set->add_detections(); - detection->set_only_required(true); - auto* location = detection->mutable_location_normalized(); - location->set_height(.1); - location->set_width(.1); - location->set_x(.4); - location->set_y(.5); - - auto input_size = ::absl::make_unique>(1000, 1000); - runner->MutableInputs() - ->Tag("VIDEO_SIZE") - .packets.push_back(Adopt(input_size.release()).At(Timestamp(0))); - - runner->MutableInputs() - ->Tag("SALIENT_REGIONS") - .packets.push_back(Adopt(detection_set.release()).At(Timestamp(0))); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("BORDERS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& static_features = output_packets[0].Get(); - CheckBorder(static_features, 1000, 1000, 495, 395); -} - -TEST(ContentZoomingCalculatorTest, ZoomTestNearOutsideBorder) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_update_rate_seconds(2); - options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2); - options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.95, .95, .05, .05), 0, runner.get()); - AddDetection(cv::Rect_(.9, .9, .1, .1), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(972, 972, 55, 55, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(958, 958, 83, 83, 1, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ZoomTestNearInsideBorder) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->mutable_kinematic_options_pan()->set_update_rate_seconds(2); - options->mutable_kinematic_options_tilt()->set_update_rate_seconds(2); - options->mutable_kinematic_options_zoom()->set_update_rate_seconds(2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(0, 0, .05, .05), 0, runner.get()); - AddDetection(cv::Rect_(0, 0, .1, .1), 1000000, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(28, 28, 55, 55, 0, runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(42, 42, 83, 83, 1, runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, VerticalShift) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_detection_shift_vertical(0.2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.1, .1, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - // 1000px * .1 offset + 1000*.1*.1 shift = 170 - CheckCropRect(150, 170, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, HorizontalShift) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_detection_shift_horizontal(0.2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.1, .1, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - // 1000px * .1 offset + 1000*.1*.1 shift = 170 - CheckCropRect(170, 150, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_detection_shift_vertical(-0.2); - options->set_detection_shift_horizontal(0.2); - auto runner = ::absl::make_unique(config); - AddDetection(cv::Rect_(.9, 0, .1, .1), 0, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(944, 56, 111, 111, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, EmptySize) { - auto config = ParseTextProtoOrDie(kConfigD); - auto runner = ::absl::make_unique(config); - MP_ASSERT_OK(runner->Run()); - ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0); -} - -TEST(ContentZoomingCalculatorTest, EmptyDetections) { - auto config = ParseTextProtoOrDie(kConfigD); - auto runner = ::absl::make_unique(config); - auto input_size = ::absl::make_unique>(1000, 1000); - runner->MutableInputs() - ->Tag("VIDEO_SIZE") - .packets.push_back(Adopt(input_size.release()).At(Timestamp(0))); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 1000, 1000, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ResolutionChangeStationary) { - auto config = ParseTextProtoOrDie(kConfigD); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1, 500, 500, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 222, 222, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 1, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ResolutionChangeStationaryWithCache) { - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache; - auto config = ParseTextProtoOrDie(kConfigD); - config.add_input_side_packet("STATE_CACHE:state_cache"); - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 222, 222, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1, 500, 500, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500 * 0.5, 500 * 0.5, 222 * 0.5, 222 * 0.5, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } -} - -TEST(ContentZoomingCalculatorTest, ResolutionChangeZooming) { - auto config = ParseTextProtoOrDie(kConfigD); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.1, .1, .8, .8), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 2000000, 500, 500, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 888, 888, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 588, 588, 1, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 2, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ResolutionChangeZoomingWithCache) { - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType cache; - auto config = ParseTextProtoOrDie(kConfigD); - config.add_input_side_packet("STATE_CACHE:state_cache"); - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetectionFrameSize(cv::Rect_(.1, .1, .8, .8), 0, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 888, 888, 0, - runner->Outputs().Tag("CROP_RECT").packets); - } - // The second runner should just resume based on state from the first runner. - { - auto runner = ::absl::make_unique(config); - runner->MutableSidePackets()->Tag("STATE_CACHE") = MakePacket< - mediapipe::autoflip::ContentZoomingCalculatorStateCacheType*>(&cache); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, - 1000, runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 2000000, 500, 500, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 588, 588, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500 * 0.5, 500 * 0.5, 288 * 0.5, 288 * 0.5, 1, - runner->Outputs().Tag("CROP_RECT").packets); - } -} - -TEST(ContentZoomingCalculatorTest, MaxZoomValue) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_max_zoom_value_deg(55); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - // 55/60 * 1000 = 916 - CheckCropRect(500, 500, 916, 916, 0, - runner->Outputs().Tag("CROP_RECT").packets); -} -#endif - -TEST(ContentZoomingCalculatorTest, MaxZoomValueOverride) { - auto config = ParseTextProtoOrDie(kConfigF); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_max_zoom_value_deg(30); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 640, 480, - runner.get(), {.max_zoom_factor_percent = 133}); - // Change resolution and allow more zoom, and give time to use the new limit - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1280, 720, - runner.get(), {.max_zoom_factor_percent = 166}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 2000000, 1280, 720, - runner.get(), {.max_zoom_factor_percent = 166}); - // Switch back to a smaller resolution with a more limited zoom - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 3000000, 640, 480, - runner.get(), {.max_zoom_factor_percent = 133}); - MP_ASSERT_OK(runner->Run()); - // Max. 133% zoomed in means min. (100/133) ~ 75% of height left: ~360 - // Max. 166% zoomed in means min. (100/166) ~ 60% of height left: ~430 - CheckCropRect(320, 240, 480, 360, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(640, 360, 769, 433, 2, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(320, 240, 480, 360, 3, - runner->Outputs().Tag("CROP_RECT").packets); -} - -#if 0 -TEST(ContentZoomingCalculatorTest, MaxZoomOutValue) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_scale_factor(1.0); - options->mutable_kinematic_options_zoom()->set_min_motion_to_reframe(5.0); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.025, .025, .95, .95), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(0, 0, -1, -1), 1000000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(0, 0, -1, -1), 2000000, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - // 55/60 * 1000 = 916 - CheckCropRect(500, 500, 950, 950, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 1000, 1000, 2, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, StartZoomedOut) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_start_zoomed_out(true); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 400000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 800000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 1000, 1000, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 880, 880, 1, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 760, 760, 2, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 655, 655, 3, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, AnimateToFirstRect) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_us_to_first_rect(1000000); - options->set_us_to_first_rect_delay(500000); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 400000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 800000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1500000, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 1000, 1000, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 1000, 1000, 1, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 470, 470, 2, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 222, 222, 3, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 222, 222, 4, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, CanControlAnimation) { - auto config = ParseTextProtoOrDie(kConfigE); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_start_zoomed_out(true); - options->set_us_to_first_rect(1000000); - options->set_us_to_first_rect_delay(500000); - auto runner = ::absl::make_unique(config); - // Request the animation for the first frame. - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get(), {.animated_zoom = true}); - // We now stop requesting animated zoom and expect the already started - // animation run to completion. This tests that the zoom in continues in the - // call when it was started in the Meet greenroom. - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 400000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 800000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1500000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 1000, 1000, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 1000, 1000, 1, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 470, 470, 2, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 222, 222, 3, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 222, 222, 4, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, DoesNotAnimateIfDisabledViaInput) { - auto config = ParseTextProtoOrDie(kConfigE); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_start_zoomed_out(true); - options->set_us_to_first_rect(1000000); - options->set_us_to_first_rect_delay(500000); - auto runner = ::absl::make_unique(config); - // Disable the animation already for the first frame. - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get(), {.animated_zoom = false}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 400000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 800000, 1000, 1000, - runner.get(), {.animated_zoom = false}); - MP_ASSERT_OK(runner->Run()); - CheckCropRect(500, 500, 1000, 1000, 0, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 880, 880, 1, - runner->Outputs().Tag("CROP_RECT").packets); - CheckCropRect(500, 500, 760, 760, 2, - runner->Outputs().Tag("CROP_RECT").packets); -} - -TEST(ContentZoomingCalculatorTest, ProvidesZeroSizeFirstRectWithoutDetections) { - auto config = ParseTextProtoOrDie(kConfigD); - auto runner = ::absl::make_unique(config); - - auto input_size = ::absl::make_unique>(1000, 1000); - runner->MutableInputs() - ->Tag("VIDEO_SIZE") - .packets.push_back(Adopt(input_size.release()).At(Timestamp(0))); - - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("FIRST_CROP_RECT").packets; - ASSERT_EQ(output_packets.size(), 1); - const auto& rect = output_packets[0].Get(); - EXPECT_EQ(rect.x_center(), 0); - EXPECT_EQ(rect.y_center(), 0); - EXPECT_EQ(rect.width(), 0); - EXPECT_EQ(rect.height(), 0); -} - -TEST(ContentZoomingCalculatorTest, ProvidesConstantFirstRect) { - auto config = ParseTextProtoOrDie(kConfigD); - auto* options = config.mutable_options()->MutableExtension( - ContentZoomingCalculatorOptions::ext); - options->set_us_to_first_rect(500000); - auto runner = ::absl::make_unique(config); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 0, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 500000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1000000, 1000, 1000, - runner.get()); - AddDetectionFrameSize(cv::Rect_(.4, .4, .2, .2), 1500000, 1000, 1000, - runner.get()); - MP_ASSERT_OK(runner->Run()); - const std::vector& output_packets = - runner->Outputs().Tag("FIRST_CROP_RECT").packets; - ASSERT_EQ(output_packets.size(), 4); - const auto& first_rect = output_packets[0].Get(); - EXPECT_NEAR(first_rect.x_center(), 0.5, 0.05); - EXPECT_NEAR(first_rect.y_center(), 0.5, 0.05); - EXPECT_NEAR(first_rect.width(), 0.222, 0.05); - EXPECT_NEAR(first_rect.height(), 0.222, 0.05); - for (int i = 1; i < 4; ++i) { - const auto& rect = output_packets[i].Get(); - EXPECT_EQ(first_rect.x_center(), rect.x_center()); - EXPECT_EQ(first_rect.y_center(), rect.y_center()); - EXPECT_EQ(first_rect.width(), rect.width()); - EXPECT_EQ(first_rect.height(), rect.height()); - } -} -#endif - -} // namespace -} // namespace autoflip - -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto deleted file mode 100644 index 92195f2a0..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/face_box_adjuster_calculator.proto +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -message FaceBoxAdjusterCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional FaceBoxAdjusterCalculatorOptions ext = 347462240; - } - - // When faces are detected in a given frame, we check these number of frames - // in the past. We include only those faces in auto framing that have been - // seen in this past history. This helps reduce False Positives and also - // handles some of the edge cases. Setting the value to 0 disables the - // feature. - optional int32 num_frame_history = 1 [default = 0]; - - // IOU threshold for matching detected faces with the faces in the frame - // history buffer. - optional float iou_threshold = 2 [default = 0.2]; - - // If true, the face boxes are adjusted based on their face pose. This is done - // to correct for extreme poses that can cause the detected face boxes to be - // either too big or too small. - optional bool adjust_for_pose = 3 [default = true]; - - // There are DEPRECATED fields. Do not use. - optional float box_area_change_per_up_tilt_degree = 4 [deprecated = true]; - optional float box_area_change_per_down_tilt_degree = 5 [deprecated = true]; - - // The ratios of the face-pose corrected IPD to the face bounding box's width - // and height respectively. - optional float ipd_face_box_width_ratio = 6 [default = 0.5566]; - optional float ipd_face_box_height_ratio = 7 [default = 0.3131]; - - // The max look up angle before considering the eye distance unstable. - optional float max_head_tilt_angle_deg = 8 [default = 5.0]; - // The min look up angle (i.e. looking down) before considering the eye - // distance unstable. - optional float min_head_tilt_angle_deg = 10 [default = -18.0]; - // The max look right angle before considering the eye distance unstable. - optional float max_head_pan_angle_deg = 11 [default = 25.0]; - // The min look right angle (i.e. looking left) before considering the eye - // distance unstable. - optional float min_head_pan_angle_deg = 12 [default = -25.0]; - - // Update rate for motion history, valid values [0.0, 1.0]. - optional float motion_history_alpha = 13 [default = 0.5]; - - // Max value of head motion (max of current or history) to be considered still - // stable. - optional float head_motion_threshold = 14 [default = 10.0]; - - // The max amount of time to use an old eye distance when the face look angle - // is unstable. - optional int32 max_facesize_history_us = 9 [default = 8000000]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc deleted file mode 100644 index 3c9aeb4c8..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.cc +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { -namespace autoflip { - -// This calculator converts detected faces to SalientRegion protos that can be -// used for downstream processing. Each SalientRegion is scored using image -// cues. Scoring can be controlled through -// FaceToRegionCalculator::scorer_options. -// Example: -// calculator: "FaceToRegionCalculator" -// input_stream: "VIDEO:frames" -// input_stream: "FACES:faces" -// output_stream: "REGIONS:regions" -// options:{ -// [mediapipe.autoflip.FaceToRegionCalculatorOptions.ext]:{ -// export_individual_face_landmarks: false -// export_whole_face: true -// } -// } -// -class FaceToRegionCalculator : public CalculatorBase { - public: - FaceToRegionCalculator(); - ~FaceToRegionCalculator() override {} - FaceToRegionCalculator(const FaceToRegionCalculator&) = delete; - FaceToRegionCalculator& operator=(const FaceToRegionCalculator&) = delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - - private: - double NormalizeX(const int pixel); - double NormalizeY(const int pixel); - // Extend the given SalientRegion to include the given point. - void ExtendSalientRegionWithPoint(const float x, const float y, - SalientRegion* region); - // Calculator options. - FaceToRegionCalculatorOptions options_; - - // A scorer used to assign weights to faces. - std::unique_ptr scorer_; - // Dimensions of video frame - int frame_width_; - int frame_height_; -}; -REGISTER_CALCULATOR(FaceToRegionCalculator); - -FaceToRegionCalculator::FaceToRegionCalculator() {} - -absl::Status FaceToRegionCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - if (cc->Inputs().HasTag("VIDEO")) { - cc->Inputs().Tag("VIDEO").Set(); - } - cc->Inputs().Tag("FACES").Set>(); - cc->Outputs().Tag("REGIONS").Set(); - return absl::OkStatus(); -} - -absl::Status FaceToRegionCalculator::Open(mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - if (!cc->Inputs().HasTag("VIDEO")) { - RET_CHECK(!options_.use_visual_scorer()) - << "VIDEO input must be provided when using visual_scorer."; - RET_CHECK(!options_.export_individual_face_landmarks()) - << "VIDEO input must be provided when export_individual_face_landmarks " - "is set true."; - RET_CHECK(!options_.export_bbox_from_landmarks()) - << "VIDEO input must be provided when export_bbox_from_landmarks " - "is set true."; - } - - scorer_ = absl::make_unique(options_.scorer_options()); - frame_width_ = -1; - frame_height_ = -1; - return absl::OkStatus(); -} - -inline double FaceToRegionCalculator::NormalizeX(const int pixel) { - return pixel / static_cast(frame_width_); -} - -inline double FaceToRegionCalculator::NormalizeY(const int pixel) { - return pixel / static_cast(frame_height_); -} - -void FaceToRegionCalculator::ExtendSalientRegionWithPoint( - const float x, const float y, SalientRegion* region) { - auto* location = region->mutable_location_normalized(); - if (!location->has_width()) { - location->set_width(NormalizeX(1)); - } else if (x < location->x()) { - location->set_width(location->width() + location->x() - x); - } else if (x > location->x() + location->width()) { - location->set_width(x - location->x()); - } - if (!location->has_height()) { - location->set_height(NormalizeY(1)); - } else if (y < location->y()) { - location->set_height(location->height() + location->y() - y); - } else if (y > location->y() + location->height()) { - location->set_height(y - location->y()); - } - - if (!location->has_x()) { - location->set_x(x); - } else { - location->set_x(std::min(location->x(), x)); - } - if (!location->has_y()) { - location->set_y(y); - } else { - location->set_y(std::min(location->y(), y)); - } -} - -absl::Status FaceToRegionCalculator::Process(mediapipe::CalculatorContext* cc) { - if (cc->Inputs().HasTag("VIDEO") && - cc->Inputs().Tag("VIDEO").Value().IsEmpty()) { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "No VIDEO input at time " << cc->InputTimestamp().Seconds(); - } - - cv::Mat frame; - if (cc->Inputs().HasTag("VIDEO")) { - frame = mediapipe::formats::MatView( - &cc->Inputs().Tag("VIDEO").Get()); - frame_width_ = frame.cols; - frame_height_ = frame.rows; - } - - auto region_set = ::absl::make_unique(); - if (!cc->Inputs().Tag("FACES").Value().IsEmpty()) { - const auto& input_faces = - cc->Inputs().Tag("FACES").Get>(); - - for (const auto& input_face : input_faces) { - RET_CHECK(input_face.location_data().format() == - mediapipe::LocationData::RELATIVE_BOUNDING_BOX) - << "Face detection input is lacking required relative_bounding_box()"; - // 6 landmarks should be provided, ordered as: - // Left eye, Right eye, Nose tip, Mouth center, Left ear tragion, Right - // ear tragion. - RET_CHECK(input_face.location_data().relative_keypoints().size() == 6) - << "Face detection input expected 6 keypoints, has " - << input_face.location_data().relative_keypoints().size(); - - const auto& location = input_face.location_data().relative_bounding_box(); - - // Reduce region size to only contain parts of the image in frame. - float x = std::max(0.0f, location.xmin()); - float y = std::max(0.0f, location.ymin()); - float width = - std::min(location.width() - abs(x - location.xmin()), 1 - x); - float height = - std::min(location.height() - abs(y - location.ymin()), 1 - y); - - // Convert the face to a region. - if (options_.export_whole_face()) { - SalientRegion* region = region_set->add_detections(); - region->mutable_location_normalized()->set_x(x); - region->mutable_location_normalized()->set_y(y); - region->mutable_location_normalized()->set_width(width); - region->mutable_location_normalized()->set_height(height); - region->mutable_signal_type()->set_standard(SignalType::FACE_FULL); - - // Score the face based on image cues. - float visual_score = 1.0f; - if (options_.use_visual_scorer()) { - MP_RETURN_IF_ERROR( - scorer_->CalculateScore(frame, *region, &visual_score)); - } - region->set_score(visual_score); - } - - // Generate two more output regions from important face landmarks. One - // includes all exterior landmarks, such as ears and chin, and the - // other includes only interior landmarks, such as the eye edges and the - // mouth. - SalientRegion core_landmark_region, all_landmark_region; - // Keypoints are ordered: Left Eye, Right Eye, Nose Tip, Mouth Center, - // Left Ear Tragion, Right Ear Tragion. - - // Set 'core' landmarks (Left Eye, Right Eye, Nose Tip, Mouth Center) - for (int i = 0; i < 4; i++) { - const auto& keypoint = input_face.location_data().relative_keypoints(i); - if (options_.export_individual_face_landmarks()) { - SalientRegion* region = region_set->add_detections(); - region->mutable_location_normalized()->set_x(keypoint.x()); - region->mutable_location_normalized()->set_y(keypoint.y()); - region->mutable_location_normalized()->set_width(NormalizeX(1)); - region->mutable_location_normalized()->set_height(NormalizeY(1)); - region->mutable_signal_type()->set_standard( - SignalType::FACE_LANDMARK); - } - - // Extend the core/full landmark regions to include the new - ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(), - &core_landmark_region); - ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(), - &all_landmark_region); - } - // Set 'all' landmarks (Left Ear Tragion, Right Ear Tragion + core) - for (int i = 4; i < 6; i++) { - const auto& keypoint = input_face.location_data().relative_keypoints(i); - if (options_.export_individual_face_landmarks()) { - SalientRegion* region = region_set->add_detections(); - region->mutable_location()->set_x(keypoint.x()); - region->mutable_location()->set_y(keypoint.y()); - region->mutable_location()->set_width(NormalizeX(1)); - region->mutable_location()->set_height(NormalizeY(1)); - region->mutable_signal_type()->set_standard( - SignalType::FACE_LANDMARK); - } - - // Extend the full landmark region to include the new landmark. - ExtendSalientRegionWithPoint(keypoint.x(), keypoint.y(), - &all_landmark_region); - } - - // Generate scores for the landmark bboxes and export them. - if (options_.export_bbox_from_landmarks() && - core_landmark_region.has_location_normalized()) { // Not empty. - float visual_score = 1.0f; - if (options_.use_visual_scorer()) { - MP_RETURN_IF_ERROR(scorer_->CalculateScore( - frame, core_landmark_region, &visual_score)); - } - core_landmark_region.set_score(visual_score); - core_landmark_region.mutable_signal_type()->set_standard( - SignalType::FACE_CORE_LANDMARKS); - *region_set->add_detections() = core_landmark_region; - } - if (options_.export_bbox_from_landmarks() && - all_landmark_region.has_location_normalized()) { // Not empty. - float visual_score = 1.0f; - if (options_.use_visual_scorer()) { - MP_RETURN_IF_ERROR(scorer_->CalculateScore(frame, all_landmark_region, - &visual_score)); - } - all_landmark_region.set_score(visual_score); - all_landmark_region.mutable_signal_type()->set_standard( - SignalType::FACE_ALL_LANDMARKS); - *region_set->add_detections() = all_landmark_region; - } - } - } - cc->Outputs().Tag("REGIONS").Add(region_set.release(), cc->InputTimestamp()); - - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto deleted file mode 100644 index 86e4c318c..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.proto +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto"; -import "mediapipe/framework/calculator.proto"; - -// Next tag: 6 -message FaceToRegionCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional FaceToRegionCalculatorOptions ext = 282401234; - } - - // Options for generating a score for the entire face from its visual - // appearance. The generated score is used to modulate the detection scores - // for whole face and/or landmark bbox region types. - optional VisualScorerOptions scorer_options = 1; - - // If true, export the large face bounding box generated by the face tracker. - // This bounding box is generally larger than the actual face and relatively - // inaccurate. - optional bool export_whole_face = 2 [default = false]; - - // If true, export a number of individual face landmarks (eyes, nose, mouth, - // ears etc) as separate SalientRegion protos. - optional bool export_individual_face_landmarks = 3 [default = false]; - - // If true, export two bounding boxes from landmarks (one for the core face - // landmarks like eyes and nose, and one for extended landmarks including ears - // and chin). - optional bool export_bbox_from_landmarks = 4 [default = true]; - - // If true, generate a score from the appearance of the face and use it to - // modulate the detection scores for whole face and/or landmark bboxes. - optional bool use_visual_scorer = 5 [default = true]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc deleted file mode 100644 index 4777c6e1d..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator_test.cc +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/face_to_region_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -using mediapipe::Detection; - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfig[] = R"( - calculator: "FaceToRegionCalculator" - input_stream: "VIDEO:frames" - input_stream: "FACES:faces" - output_stream: "REGIONS:regions" - )"; - -const char kConfigNoVideo[] = R"( - calculator: "FaceToRegionCalculator" - input_stream: "FACES:faces" - output_stream: "REGIONS:regions" - )"; - -const char kFace1[] = R"(location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: -0.00375 - ymin: 0.003333 - width: 0.125 - height: 0.33333 - } - relative_keypoints { x: 0.03125 y: 0.05 } - relative_keypoints { x: 0.0875 y: 0.0666666 } - relative_keypoints { x: 0.03125 y: 0.05 } - relative_keypoints { x: 0.0875 y: 0.0666666 } - relative_keypoints { x: 0.0250 y: 0.0666666 } - relative_keypoints { x: 0.0950 y: 0.0666666 } - })"; - -const char kFace2[] = R"(location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: 0.0025 - ymin: 0.005 - width: 0.25 - height: 0.5 - } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - })"; - -const char kFace3[] = R"(location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: 0.0 - ymin: 0.0 - width: 0.5 - height: 0.5 - } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - relative_keypoints { x: 0 y: 0 } - })"; - -void SetInputs(const std::vector& faces, const bool include_video, - CalculatorRunner* runner) { - // Setup an input video frame. - if (include_video) { - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, 800, 600); - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp::PostStream())); - } - // Setup two faces as input. - auto input_faces = ::absl::make_unique>(); - // A face with landmarks. - for (const auto& face : faces) { - input_faces->push_back(ParseTextProtoOrDie(face)); - } - runner->MutableInputs()->Tag("FACES").packets.push_back( - Adopt(input_faces.release()).At(Timestamp::PostStream())); -} - -CalculatorGraphConfig::Node MakeConfig(std::string base_config, bool whole_face, - bool landmarks, bool bb_from_landmarks, - bool visual_scoring) { - auto config = ParseTextProtoOrDie(base_config); - config.mutable_options() - ->MutableExtension(FaceToRegionCalculatorOptions::ext) - ->set_export_whole_face(whole_face); - config.mutable_options() - ->MutableExtension(FaceToRegionCalculatorOptions::ext) - ->set_export_individual_face_landmarks(landmarks); - config.mutable_options() - ->MutableExtension(FaceToRegionCalculatorOptions::ext) - ->set_export_bbox_from_landmarks(bb_from_landmarks); - config.mutable_options() - ->MutableExtension(FaceToRegionCalculatorOptions::ext) - ->set_use_visual_scorer(visual_scoring); - - return config; -} - -TEST(FaceToRegionCalculatorTest, FaceFullTypeSize) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfig, true, false, false, true)); - SetInputs({kFace1, kFace2}, true, runner.get()); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(2, regions.detections().size()); - auto face_1 = regions.detections(0); - EXPECT_EQ(face_1.signal_type().standard(), SignalType::FACE_FULL); - EXPECT_FLOAT_EQ(face_1.location_normalized().x(), 0); - EXPECT_FLOAT_EQ(face_1.location_normalized().y(), 0.003333); - EXPECT_FLOAT_EQ(face_1.location_normalized().width(), 0.12125); - EXPECT_FLOAT_EQ(face_1.location_normalized().height(), 0.33333); - EXPECT_FLOAT_EQ(face_1.score(), 0.040214583); - - auto face_2 = regions.detections(1); - EXPECT_EQ(face_2.signal_type().standard(), SignalType::FACE_FULL); - EXPECT_FLOAT_EQ(face_2.location_normalized().x(), 0.0025); - EXPECT_FLOAT_EQ(face_2.location_normalized().y(), 0.005); - EXPECT_FLOAT_EQ(face_2.location_normalized().width(), 0.25); - EXPECT_FLOAT_EQ(face_2.location_normalized().height(), 0.5); - EXPECT_FLOAT_EQ(face_2.score(), 0.125); -} - -TEST(FaceToRegionCalculatorTest, FaceLandmarksTypeSize) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfig, false, true, false, true)); - SetInputs({kFace1}, true, runner.get()); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(6, regions.detections().size()); - auto landmark_1 = regions.detections(0); - EXPECT_EQ(landmark_1.signal_type().standard(), SignalType::FACE_LANDMARK); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.00125); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.0016666667); - - auto landmark_2 = regions.detections(1); - EXPECT_EQ(landmark_2.signal_type().standard(), SignalType::FACE_LANDMARK); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.0875); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.0666666); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.00125); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.0016666667); -} - -TEST(FaceToRegionCalculatorTest, FaceLandmarksBox) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfig, false, false, true, true)); - SetInputs({kFace1}, true, runner.get()); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(2, regions.detections().size()); - auto landmark_1 = regions.detections(0); - EXPECT_EQ(landmark_1.signal_type().standard(), - SignalType::FACE_CORE_LANDMARKS); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().x(), 0.03125); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().y(), 0.05); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().width(), 0.056249999); - EXPECT_FLOAT_EQ(landmark_1.location_normalized().height(), 0.016666602); - EXPECT_FLOAT_EQ(landmark_1.score(), 0.00084375002); - - auto landmark_2 = regions.detections(1); - EXPECT_EQ(landmark_2.signal_type().standard(), - SignalType::FACE_ALL_LANDMARKS); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().x(), 0.025); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().y(), 0.050000001); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().width(), 0.07); - EXPECT_FLOAT_EQ(landmark_2.location_normalized().height(), 0.016666602); - EXPECT_FLOAT_EQ(landmark_2.score(), 0.00105); -} - -TEST(FaceToRegionCalculatorTest, FaceScore) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfig, true, false, false, true)); - SetInputs({kFace3}, true, runner.get()); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(1, regions.detections().size()); - auto landmark_1 = regions.detections(0); - EXPECT_FLOAT_EQ(landmark_1.score(), 0.25); -} - -TEST(FaceToRegionCalculatorTest, FaceNoVideoVisualScoreFail) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfigNoVideo, true, false, false, true)); - SetInputs({kFace3}, false, runner.get()); - - // Run the calculator. - ASSERT_FALSE(runner->Run().ok()); -} - -TEST(FaceToRegionCalculatorTest, FaceNoVideoLandmarksFail) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfigNoVideo, false, true, false, false)); - SetInputs({kFace3}, false, runner.get()); - - // Run the calculator. - ASSERT_FALSE(runner->Run().ok()); -} - -TEST(FaceToRegionCalculatorTest, FaceNoVideoBBLandmarksFail) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfigNoVideo, false, false, true, false)); - SetInputs({kFace3}, false, runner.get()); - - // Run the calculator. - ASSERT_FALSE(runner->Run().ok()); -} - -TEST(FaceToRegionCalculatorTest, FaceNoVideoPass) { - // Setup test - auto runner = ::absl::make_unique( - MakeConfig(kConfigNoVideo, true, false, false, false)); - SetInputs({kFace1, kFace2}, false, runner.get()); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(2, regions.detections().size()); - auto face_1 = regions.detections(0); - EXPECT_EQ(face_1.signal_type().standard(), SignalType::FACE_FULL); - EXPECT_FLOAT_EQ(face_1.location_normalized().x(), 0); - EXPECT_FLOAT_EQ(face_1.location_normalized().y(), 0.003333); - EXPECT_FLOAT_EQ(face_1.location_normalized().width(), 0.12125); - EXPECT_FLOAT_EQ(face_1.location_normalized().height(), 0.33333); - EXPECT_FLOAT_EQ(face_1.score(), 1); - - auto face_2 = regions.detections(1); - EXPECT_EQ(face_2.signal_type().standard(), SignalType::FACE_FULL); - EXPECT_FLOAT_EQ(face_2.location_normalized().x(), 0.0025); - EXPECT_FLOAT_EQ(face_2.location_normalized().y(), 0.005); - EXPECT_FLOAT_EQ(face_2.location_normalized().width(), 0.25); - EXPECT_FLOAT_EQ(face_2.location_normalized().height(), 0.5); - EXPECT_FLOAT_EQ(face_2.score(), 1); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc deleted file mode 100644 index 80f0f4552..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.cc +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// This calculator converts detections from ObjectLocalizationCalculator to -// SalientRegion protos that can be used for downstream processing. -class LocalizationToRegionCalculator : public mediapipe::CalculatorBase { - public: - LocalizationToRegionCalculator(); - ~LocalizationToRegionCalculator() override {} - LocalizationToRegionCalculator(const LocalizationToRegionCalculator&) = - delete; - LocalizationToRegionCalculator& operator=( - const LocalizationToRegionCalculator&) = delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - - private: - // Calculator options. - LocalizationToRegionCalculatorOptions options_; -}; -REGISTER_CALCULATOR(LocalizationToRegionCalculator); - -LocalizationToRegionCalculator::LocalizationToRegionCalculator() {} - -namespace { - -// Converts an object detection to a autoflip SignalType. Returns true if the -// std::string label has a autoflip label. -bool MatchType(const std::string& label, SignalType* type) { - if (label == "person") { - type->set_standard(SignalType::HUMAN); - return true; - } - if (label == "car" || label == "truck") { - type->set_standard(SignalType::CAR); - return true; - } - if (label == "dog" || label == "cat" || label == "bird" || label == "horse") { - type->set_standard(SignalType::PET); - return true; - } - return false; -} - -// Converts a detection to a SalientRegion with a given label. -void FillSalientRegion(const mediapipe::Detection& detection, - const SignalType& label, SalientRegion* region) { - const auto& location = detection.location_data().relative_bounding_box(); - region->mutable_location_normalized()->set_x(location.xmin()); - region->mutable_location_normalized()->set_y(location.ymin()); - region->mutable_location_normalized()->set_width(location.width()); - region->mutable_location_normalized()->set_height(location.height()); - region->set_score(1.0); - *region->mutable_signal_type() = label; -} - -} // namespace - -absl::Status LocalizationToRegionCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - cc->Inputs().Tag("DETECTIONS").Set>(); - cc->Outputs().Tag("REGIONS").Set(); - return absl::OkStatus(); -} - -absl::Status LocalizationToRegionCalculator::Open( - mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - - return absl::OkStatus(); -} - -absl::Status LocalizationToRegionCalculator::Process( - mediapipe::CalculatorContext* cc) { - const auto& annotations = - cc->Inputs().Tag("DETECTIONS").Get>(); - auto regions = ::absl::make_unique(); - for (const auto& detection : annotations) { - RET_CHECK_EQ(detection.label().size(), 1) - << "Number of labels not equal to one."; - SignalType autoflip_label; - if (MatchType(detection.label(0), &autoflip_label) && - options_.output_standard_signals()) { - FillSalientRegion(detection, autoflip_label, regions->add_detections()); - } - if (options_.output_all_signals()) { - SignalType object; - object.set_standard(SignalType::OBJECT); - FillSalientRegion(detection, object, regions->add_detections()); - } - } - - cc->Outputs().Tag("REGIONS").Add(regions.release(), cc->InputTimestamp()); - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto deleted file mode 100644 index 1dd00acfb..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -message LocalizationToRegionCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional LocalizationToRegionCalculatorOptions ext = 284226721; - } - - // Output standard autoflip signals only (Human, Pet, Car, etc) and apply - // standard autoflip labels. - optional bool output_standard_signals = 1 [default = true]; - // Output all signals (regardless of label) and set autoflip label as - // 'Object'. Can be combined with output_standard_signals giving each - // detection a 'object' label and a autoflip sepcific label. - optional bool output_all_signals = 2 [default = false]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc deleted file mode 100644 index d7b06c57c..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator_test.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/localization_to_region_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -using mediapipe::Detection; - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfig[] = R"( - calculator: "LocalizationToRegionCalculator" - input_stream: "DETECTIONS:detections" - output_stream: "REGIONS:regions" - )"; - -const char kCar[] = R"( - label: "car" - location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: -0.00375 - ymin: 0.003333 - width: 0.125 - height: 0.33333 - } - })"; - -const char kDog[] = R"( - label: "dog" - location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: 0.0025 - ymin: 0.005 - width: 0.25 - height: 0.5 - } - })"; - -const char kZebra[] = R"( - label: "zebra" - location_data { - format: RELATIVE_BOUNDING_BOX - relative_bounding_box { - xmin: 0.0 - ymin: 0.0 - width: 0.5 - height: 0.5 - } - })"; - -void SetInputs(CalculatorRunner* runner, - const std::vector& detections) { - auto inputs = ::absl::make_unique>(); - // A face with landmarks. - for (const auto& detection : detections) { - inputs->push_back(ParseTextProtoOrDie(detection)); - } - runner->MutableInputs() - ->Tag("DETECTIONS") - .packets.push_back(Adopt(inputs.release()).At(Timestamp::PostStream())); -} - -CalculatorGraphConfig::Node MakeConfig(bool output_standard, bool output_all) { - auto config = ParseTextProtoOrDie(kConfig); - - config.mutable_options() - ->MutableExtension(LocalizationToRegionCalculatorOptions::ext) - ->set_output_standard_signals(output_standard); - - config.mutable_options() - ->MutableExtension(LocalizationToRegionCalculatorOptions::ext) - ->set_output_all_signals(output_all); - - return config; -} - -TEST(LocalizationToRegionCalculatorTest, StandardTypes) { - // Setup test - auto runner = ::absl::make_unique(MakeConfig(true, false)); - SetInputs(runner.get(), {kCar, kDog, kZebra}); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(2, regions.detections().size()); - const auto& detection = regions.detections(0); - EXPECT_EQ(detection.signal_type().standard(), SignalType::CAR); - EXPECT_FLOAT_EQ(detection.location_normalized().x(), -0.00375); - EXPECT_FLOAT_EQ(detection.location_normalized().y(), 0.003333); - EXPECT_FLOAT_EQ(detection.location_normalized().width(), 0.125); - EXPECT_FLOAT_EQ(detection.location_normalized().height(), 0.33333); - const auto& detection_1 = regions.detections(1); - EXPECT_EQ(detection_1.signal_type().standard(), SignalType::PET); - EXPECT_FLOAT_EQ(detection_1.location_normalized().x(), 0.0025); - EXPECT_FLOAT_EQ(detection_1.location_normalized().y(), 0.005); - EXPECT_FLOAT_EQ(detection_1.location_normalized().width(), 0.25); - EXPECT_FLOAT_EQ(detection_1.location_normalized().height(), 0.5); -} - -TEST(LocalizationToRegionCalculatorTest, AllTypes) { - // Setup test - auto runner = ::absl::make_unique(MakeConfig(false, true)); - SetInputs(runner.get(), {kCar, kDog, kZebra}); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(3, regions.detections().size()); -} - -TEST(LocalizationToRegionCalculatorTest, BothTypes) { - // Setup test - auto runner = ::absl::make_unique(MakeConfig(true, true)); - SetInputs(runner.get(), {kCar, kDog, kZebra}); - - // Run the calculator. - MP_ASSERT_OK(runner->Run()); - - // Check the output regions. - const std::vector& output_packets = - runner->Outputs().Tag("REGIONS").packets; - ASSERT_EQ(1, output_packets.size()); - const auto& regions = output_packets[0].Get(); - ASSERT_EQ(5, regions.detections().size()); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc deleted file mode 100644 index 885753d63..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc +++ /dev/null @@ -1,822 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h" - -#include - -#include "absl/memory/memory.h" -#include "absl/strings/str_format.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h" -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { -namespace autoflip { - -constexpr char kInputVideoFrames[] = "VIDEO_FRAMES"; -constexpr char kInputVideoSize[] = "VIDEO_SIZE"; -constexpr char kInputKeyFrames[] = "KEY_FRAMES"; -constexpr char kInputDetections[] = "DETECTION_FEATURES"; -constexpr char kInputStaticFeatures[] = "STATIC_FEATURES"; -constexpr char kInputShotBoundaries[] = "SHOT_BOUNDARIES"; -constexpr char kInputExternalSettings[] = "EXTERNAL_SETTINGS"; -// This side packet must be used in conjunction with -// TargetSizeType::MAXIMIZE_TARGET_DIMENSION -constexpr char kAspectRatio[] = "EXTERNAL_ASPECT_RATIO"; - -// Output the cropped frames, as well as visualization of crop regions and focus -// points. Note that, KEY_FRAME_CROP_REGION_VIZ_FRAMES and -// SALIENT_POINT_FRAME_VIZ_FRAMES can only be enabled when CROPPED_FRAMES is -// enabled. -constexpr char kOutputCroppedFrames[] = "CROPPED_FRAMES"; -// Shows detections on key frames. Any static borders will be removed from the -// output frame. -constexpr char kOutputKeyFrameCropViz[] = "KEY_FRAME_CROP_REGION_VIZ_FRAMES"; -// Shows x/y (raw unsmoothed) cropping and focus points. Any static borders -// will be removed from the output frame. -constexpr char kOutputFocusPointFrameViz[] = "SALIENT_POINT_FRAME_VIZ_FRAMES"; -// Shows final smoothed cropping and a focused area of the camera. Any static -// borders will remain and be shown in grey. Output frame will match input -// frame size. -constexpr char kOutputFramingAndDetections[] = "FRAMING_DETECTIONS_VIZ_FRAMES"; -// Final summary of cropping. -constexpr char kOutputSummary[] = "CROPPING_SUMMARY"; - -// External rendering outputs -constexpr char kExternalRenderingPerFrame[] = "EXTERNAL_RENDERING_PER_FRAME"; -constexpr char kExternalRenderingFullVid[] = "EXTERNAL_RENDERING_FULL_VID"; - -absl::Status SceneCroppingCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - if (cc->InputSidePackets().HasTag(kInputExternalSettings)) { - cc->InputSidePackets().Tag(kInputExternalSettings).Set(); - } - if (cc->InputSidePackets().HasTag(kAspectRatio)) { - cc->InputSidePackets().Tag(kAspectRatio).Set(); - } - if (cc->Inputs().HasTag(kInputVideoFrames)) { - cc->Inputs().Tag(kInputVideoFrames).Set(); - } - if (cc->Inputs().HasTag(kInputVideoSize)) { - cc->Inputs().Tag(kInputVideoSize).Set>(); - } - if (cc->Inputs().HasTag(kInputKeyFrames)) { - cc->Inputs().Tag(kInputKeyFrames).Set(); - } - cc->Inputs().Tag(kInputDetections).Set(); - if (cc->Inputs().HasTag(kInputStaticFeatures)) { - cc->Inputs().Tag(kInputStaticFeatures).Set(); - } - if (cc->Inputs().HasTag(kInputShotBoundaries)) { - cc->Inputs().Tag(kInputShotBoundaries).Set(); - } - - if (cc->Outputs().HasTag(kOutputCroppedFrames)) { - cc->Outputs().Tag(kOutputCroppedFrames).Set(); - } - if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) { - RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames)) - << "KEY_FRAME_CROP_REGION_VIZ_FRAMES can only be used when " - "CROPPED_FRAMES is specified."; - cc->Outputs().Tag(kOutputKeyFrameCropViz).Set(); - } - if (cc->Outputs().HasTag(kOutputFramingAndDetections)) { - RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames)) - << "FRAMING_DETECTIONS_VIZ_FRAMES can only be used when " - "CROPPED_FRAMES is specified."; - cc->Outputs().Tag(kOutputFramingAndDetections).Set(); - } - if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) { - RET_CHECK(cc->Outputs().HasTag(kOutputCroppedFrames)) - << "SALIENT_POINT_FRAME_VIZ_FRAMES can only be used when " - "CROPPED_FRAMES is specified."; - cc->Outputs().Tag(kOutputFocusPointFrameViz).Set(); - } - if (cc->Outputs().HasTag(kOutputSummary)) { - cc->Outputs().Tag(kOutputSummary).Set(); - } - if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) { - cc->Outputs().Tag(kExternalRenderingPerFrame).Set(); - } - if (cc->Outputs().HasTag(kExternalRenderingFullVid)) { - cc->Outputs() - .Tag(kExternalRenderingFullVid) - .Set>(); - } - RET_CHECK(cc->Inputs().HasTag(kInputVideoFrames) ^ - cc->Inputs().HasTag(kInputVideoSize)) - << "VIDEO_FRAMES or VIDEO_SIZE must be set and not both."; - RET_CHECK(!(cc->Inputs().HasTag(kInputVideoSize) && - cc->Inputs().HasTag(kOutputCroppedFrames))) - << "CROPPED_FRAMES (internal cropping) has been set as an output without " - "VIDEO_FRAMES (video data) input."; - RET_CHECK(cc->Outputs().HasTag(kExternalRenderingPerFrame) || - cc->Outputs().HasTag(kExternalRenderingFullVid) || - cc->Outputs().HasTag(kOutputCroppedFrames)) - << "At leaset one output stream must be specified"; - return absl::OkStatus(); -} - -absl::Status SceneCroppingCalculator::Open(CalculatorContext* cc) { - options_ = cc->Options(); - RET_CHECK_GT(options_.max_scene_size(), 0) - << "Maximum scene size is non-positive."; - RET_CHECK_GE(options_.prior_frame_buffer_size(), 0) - << "Prior frame buffer size is negative."; - - RET_CHECK(options_.solid_background_frames_padding_fraction() >= 0.0 && - options_.solid_background_frames_padding_fraction() <= 1.0) - << "Solid background frames padding fraction is not in [0, 1]."; - const auto& padding_params = options_.padding_parameters(); - background_contrast_ = padding_params.background_contrast(); - RET_CHECK(background_contrast_ >= 0.0 && background_contrast_ <= 1.0) - << "Background contrast " << background_contrast_ << " is not in [0, 1]."; - blur_cv_size_ = padding_params.blur_cv_size(); - RET_CHECK_GT(blur_cv_size_, 0) << "Blur cv size is non-positive."; - overlay_opacity_ = padding_params.overlay_opacity(); - RET_CHECK(overlay_opacity_ >= 0.0 && overlay_opacity_ <= 1.0) - << "Overlay opacity " << overlay_opacity_ << " is not in [0, 1]."; - - // Set default camera model to polynomial_path_solver. - if (!options_.camera_motion_options().has_kinematic_options()) { - options_.mutable_camera_motion_options() - ->mutable_polynomial_path_solver() - ->set_prior_frame_buffer_size(options_.prior_frame_buffer_size()); - } - if (cc->Outputs().HasTag(kOutputSummary)) { - summary_ = absl::make_unique(); - } - if (cc->Outputs().HasTag(kExternalRenderingFullVid)) { - external_render_list_ = - absl::make_unique>(); - } - should_perform_frame_cropping_ = cc->Outputs().HasTag(kOutputCroppedFrames); - scene_camera_motion_analyzer_ = absl::make_unique( - options_.scene_camera_motion_analyzer_options()); - return absl::OkStatus(); -} - -namespace { -absl::Status ParseAspectRatioString(const std::string& aspect_ratio_string, - double* aspect_ratio) { - std::string error_msg = - "Aspect ratio std::string must be in the format of 'width:height', e.g. " - "'1:1' or '5:4', your input was " + - aspect_ratio_string; - auto pos = aspect_ratio_string.find(':'); - RET_CHECK(pos != std::string::npos) << error_msg; - double width_ratio; - RET_CHECK(absl::SimpleAtod(aspect_ratio_string.substr(0, pos), &width_ratio)) - << error_msg; - double height_ratio; - RET_CHECK(absl::SimpleAtod( - aspect_ratio_string.substr(pos + 1, aspect_ratio_string.size()), - &height_ratio)) - << error_msg; - *aspect_ratio = width_ratio / height_ratio; - return absl::OkStatus(); -} -void ConstructExternalRenderMessage( - const cv::Rect& crop_from_location, const cv::Rect& render_to_location, - const cv::Scalar& padding_color, const uint64 timestamp_us, - ExternalRenderFrame* external_render_message) { - auto crop_from_message = - external_render_message->mutable_crop_from_location(); - crop_from_message->set_x(crop_from_location.x); - crop_from_message->set_y(crop_from_location.y); - crop_from_message->set_width(crop_from_location.width); - crop_from_message->set_height(crop_from_location.height); - auto render_to_message = - external_render_message->mutable_render_to_location(); - render_to_message->set_x(render_to_location.x); - render_to_message->set_y(render_to_location.y); - render_to_message->set_width(render_to_location.width); - render_to_message->set_height(render_to_location.height); - auto padding_color_message = external_render_message->mutable_padding_color(); - padding_color_message->set_r(padding_color[0]); - padding_color_message->set_g(padding_color[1]); - padding_color_message->set_b(padding_color[2]); - external_render_message->set_timestamp_us(timestamp_us); -} - -double GetRatio(int width, int height) { - return static_cast(width) / height; -} - -int RoundToEven(float value) { - int rounded_value = std::round(value); - if (rounded_value % 2 == 1) { - rounded_value = std::max(2, rounded_value - 1); - } - return rounded_value; -} - -} // namespace - -absl::Status SceneCroppingCalculator::InitializeSceneCroppingCalculator( - mediapipe::CalculatorContext* cc) { - if (cc->Inputs().HasTag(kInputVideoFrames)) { - const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get(); - frame_width_ = frame.Width(); - frame_height_ = frame.Height(); - frame_format_ = frame.Format(); - } else if (cc->Inputs().HasTag(kInputVideoSize)) { - frame_width_ = - cc->Inputs().Tag(kInputVideoSize).Get>().first; - frame_height_ = - cc->Inputs().Tag(kInputVideoSize).Get>().second; - } else { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Input VIDEO or VIDEO_SIZE must be provided."; - } - RET_CHECK_GT(frame_height_, 0) << "Input frame height is non-positive."; - RET_CHECK_GT(frame_width_, 0) << "Input frame width is non-positive."; - - // Calculate target width and height. - switch (options_.target_size_type()) { - case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT: - RET_CHECK(options_.has_target_width() && options_.has_target_height()) - << "Target width and height have to be specified."; - target_height_ = RoundToEven(frame_height_); - target_width_ = - RoundToEven(target_height_ * GetRatio(options_.target_width(), - options_.target_height())); - break; - case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH: - RET_CHECK(options_.has_target_width() && options_.has_target_height()) - << "Target width and height have to be specified."; - target_width_ = RoundToEven(frame_width_); - target_height_ = - RoundToEven(target_width_ / GetRatio(options_.target_width(), - options_.target_height())); - break; - case SceneCroppingCalculatorOptions::MAXIMIZE_TARGET_DIMENSION: { - RET_CHECK(cc->InputSidePackets().HasTag(kAspectRatio)) - << "MAXIMIZE_TARGET_DIMENSION is set without an " - "external_aspect_ratio"; - double requested_aspect_ratio; - MP_RETURN_IF_ERROR(ParseAspectRatioString( - cc->InputSidePackets().Tag(kAspectRatio).Get(), - &requested_aspect_ratio)); - const double original_aspect_ratio = - GetRatio(frame_width_, frame_height_); - if (original_aspect_ratio > requested_aspect_ratio) { - target_height_ = RoundToEven(frame_height_); - target_width_ = RoundToEven(target_height_ * requested_aspect_ratio); - } else { - target_width_ = RoundToEven(frame_width_); - target_height_ = RoundToEven(target_width_ / requested_aspect_ratio); - } - break; - } - case SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION: - RET_CHECK(options_.has_target_width() && options_.has_target_height()) - << "Target width and height have to be specified."; - target_width_ = options_.target_width(); - target_height_ = options_.target_height(); - break; - case SceneCroppingCalculatorOptions::KEEP_ORIGINAL_DIMENSION: - target_width_ = frame_width_; - target_height_ = frame_height_; - break; - case SceneCroppingCalculatorOptions::UNKNOWN: - return absl::InvalidArgumentError("target_size_type not set properly."); - } - target_aspect_ratio_ = GetRatio(target_width_, target_height_); - - // Set keyframe width/height for feature upscaling. - RET_CHECK(!(cc->Inputs().HasTag(kInputKeyFrames) && - (options_.has_video_features_width() || - options_.has_video_features_height()))) - << "Key frame size must be defined by either providing the input stream " - "KEY_FRAMES or setting video_features_width/video_features_height as " - "calculator options. Both methods cannot be used together."; - if (options_.has_video_features_width() && - options_.has_video_features_height()) { - key_frame_width_ = options_.video_features_width(); - key_frame_height_ = options_.video_features_height(); - } else if (!cc->Inputs().HasTag(kInputKeyFrames)) { - key_frame_width_ = frame_width_; - key_frame_height_ = frame_height_; - } - // Check provided dimensions. - RET_CHECK_GT(target_width_, 0) << "Target width is non-positive."; - // TODO: it seems this check is too strict and maybe limiting, - // considering the receiver of frames can be something other than encoder. - RET_CHECK_NE(target_width_ % 2, 1) - << "Target width cannot be odd, because encoder expects dimension " - "values to be even."; - RET_CHECK_GT(target_height_, 0) << "Target height is non-positive."; - RET_CHECK_NE(target_height_ % 2, 1) - << "Target height cannot be odd, because encoder expects dimension " - "values to be even."; - - scene_cropper_ = absl::make_unique( - options_.camera_motion_options(), frame_width_, frame_height_); - - return absl::OkStatus(); -} - -bool HasFrameSignal(mediapipe::CalculatorContext* cc) { - if (cc->Inputs().HasTag(kInputVideoFrames)) { - return !cc->Inputs().Tag(kInputVideoFrames).Value().IsEmpty(); - } - return !cc->Inputs().Tag(kInputVideoSize).Value().IsEmpty(); -} - -absl::Status SceneCroppingCalculator::Process( - mediapipe::CalculatorContext* cc) { - // Sets frame dimension and initializes scenecroppingcalculator on first video - // frame. - if (frame_width_ < 0) { - MP_RETURN_IF_ERROR(InitializeSceneCroppingCalculator(cc)); - } - - // Sets key frame dimension on first keyframe. - if (cc->Inputs().HasTag(kInputKeyFrames) && - !cc->Inputs().Tag(kInputKeyFrames).Value().IsEmpty() && - key_frame_width_ < 0) { - const auto& key_frame = cc->Inputs().Tag(kInputKeyFrames).Get(); - key_frame_width_ = key_frame.Width(); - key_frame_height_ = key_frame.Height(); - } - - // Processes a scene when shot boundary or buffer is full. - bool is_end_of_scene = false; - if (cc->Inputs().HasTag(kInputShotBoundaries) && - !cc->Inputs().Tag(kInputShotBoundaries).Value().IsEmpty()) { - is_end_of_scene = cc->Inputs().Tag(kInputShotBoundaries).Get(); - } - - if (!scene_frame_timestamps_.empty() && (is_end_of_scene)) { - continue_last_scene_ = false; - MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc)); - } - - // Saves frame and timestamp and whether it is a key frame. - if (HasFrameSignal(cc)) { - // Only buffer frames if |should_perform_frame_cropping_| is true. - if (should_perform_frame_cropping_) { - const auto& frame = cc->Inputs().Tag(kInputVideoFrames).Get(); - const cv::Mat frame_mat = formats::MatView(&frame); - cv::Mat copy_mat; - frame_mat.copyTo(copy_mat); - scene_frames_or_empty_.push_back(copy_mat); - } - scene_frame_timestamps_.push_back(cc->InputTimestamp().Value()); - is_key_frames_.push_back( - !cc->Inputs().Tag(kInputDetections).Value().IsEmpty()); - } - - // Packs key frame info. - if (!cc->Inputs().Tag(kInputDetections).Value().IsEmpty()) { - const auto& detections = - cc->Inputs().Tag(kInputDetections).Get(); - KeyFrameInfo key_frame_info; - MP_RETURN_IF_ERROR(PackKeyFrameInfo( - cc->InputTimestamp().Value(), detections, frame_width_, frame_height_, - key_frame_width_, key_frame_height_, &key_frame_info)); - key_frame_infos_.push_back(key_frame_info); - } - - // Buffers static features. - if (cc->Inputs().HasTag(kInputStaticFeatures) && - !cc->Inputs().Tag(kInputStaticFeatures).Value().IsEmpty()) { - static_features_.push_back( - cc->Inputs().Tag(kInputStaticFeatures).Get()); - static_features_timestamps_.push_back(cc->InputTimestamp().Value()); - } - - const bool force_buffer_flush = - scene_frame_timestamps_.size() >= options_.max_scene_size(); - if (!scene_frame_timestamps_.empty() && force_buffer_flush) { - MP_RETURN_IF_ERROR(ProcessScene(is_end_of_scene, cc)); - continue_last_scene_ = true; - } - - return absl::OkStatus(); -} - -absl::Status SceneCroppingCalculator::Close(mediapipe::CalculatorContext* cc) { - if (!scene_frame_timestamps_.empty()) { - MP_RETURN_IF_ERROR(ProcessScene(/* is_end_of_scene = */ true, cc)); - } - if (cc->Outputs().HasTag(kOutputSummary)) { - cc->Outputs() - .Tag(kOutputSummary) - .Add(summary_.release(), Timestamp::PostStream()); - } - if (cc->Outputs().HasTag(kExternalRenderingFullVid)) { - cc->Outputs() - .Tag(kExternalRenderingFullVid) - .Add(external_render_list_.release(), Timestamp::PostStream()); - } - return absl::OkStatus(); -} - -// TODO: split this function into two, one for calculating the border -// sizes, the other for the actual removal of borders from the frames. -absl::Status SceneCroppingCalculator::RemoveStaticBorders( - CalculatorContext* cc, int* top_border_size, int* bottom_border_size) { - *top_border_size = 0; - *bottom_border_size = 0; - MP_RETURN_IF_ERROR(ComputeSceneStaticBordersSize( - static_features_, top_border_size, bottom_border_size)); - const double scale = static_cast(frame_height_) / key_frame_height_; - top_border_distance_ = std::round(scale * *top_border_size); - const int bottom_border_distance = std::round(scale * *bottom_border_size); - effective_frame_height_ = - frame_height_ - top_border_distance_ - bottom_border_distance; - - // Store shallow copy of the original frames for debug display if required - // before static areas are removed. - if (cc->Outputs().HasTag(kOutputFramingAndDetections)) { - raw_scene_frames_or_empty_ = {scene_frames_or_empty_.begin(), - scene_frames_or_empty_.end()}; - } - - if (top_border_distance_ > 0 || bottom_border_distance > 0) { - VLOG(1) << "Remove top border " << top_border_distance_ << " bottom border " - << bottom_border_distance; - // Remove borders from frames. - cv::Rect roi(0, top_border_distance_, frame_width_, - effective_frame_height_); - for (int i = 0; i < scene_frames_or_empty_.size(); ++i) { - cv::Mat tmp; - scene_frames_or_empty_[i](roi).copyTo(tmp); - scene_frames_or_empty_[i] = tmp; - } - // Adjust detection bounding boxes. - for (int i = 0; i < key_frame_infos_.size(); ++i) { - DetectionSet adjusted_detections; - const auto& detections = key_frame_infos_[i].detections(); - for (int j = 0; j < detections.detections_size(); ++j) { - const auto& detection = detections.detections(j); - SalientRegion adjusted_detection = detection; - // Clamp the box to be within the de-bordered frame. - if (!ClampRect(0, top_border_distance_, frame_width_, - top_border_distance_ + effective_frame_height_, - adjusted_detection.mutable_location()) - .ok()) { - continue; - } - // Offset the y position. - adjusted_detection.mutable_location()->set_y( - adjusted_detection.location().y() - top_border_distance_); - *adjusted_detections.add_detections() = adjusted_detection; - } - *key_frame_infos_[i].mutable_detections() = adjusted_detections; - } - } - return absl::OkStatus(); -} - -absl::Status SceneCroppingCalculator::InitializeFrameCropRegionComputer() { - key_frame_crop_options_ = options_.key_frame_crop_options(); - MP_RETURN_IF_ERROR( - SetKeyFrameCropTarget(frame_width_, effective_frame_height_, - target_aspect_ratio_, &key_frame_crop_options_)); - VLOG(1) << "Target width " << key_frame_crop_options_.target_width(); - VLOG(1) << "Target height " << key_frame_crop_options_.target_height(); - frame_crop_region_computer_ = - absl::make_unique(key_frame_crop_options_); - return absl::OkStatus(); -} - -void SceneCroppingCalculator::FilterKeyFrameInfo() { - if (!options_.user_hint_override()) { - return; - } - std::vector user_hints_only; - bool has_user_hints = false; - for (auto key_frame : key_frame_infos_) { - DetectionSet user_hint_only_set; - for (const auto& detection : key_frame.detections().detections()) { - if (detection.signal_type().has_standard() && - detection.signal_type().standard() == SignalType::USER_HINT) { - *user_hint_only_set.add_detections() = detection; - has_user_hints = true; - } - } - *key_frame.mutable_detections() = user_hint_only_set; - user_hints_only.push_back(key_frame); - } - if (has_user_hints) { - key_frame_infos_ = user_hints_only; - } -} - -absl::Status SceneCroppingCalculator::ProcessScene(const bool is_end_of_scene, - CalculatorContext* cc) { - // Removes detections under special circumstances. - FilterKeyFrameInfo(); - - // Removes any static borders. - int top_static_border_size, bottom_static_border_size; - MP_RETURN_IF_ERROR(RemoveStaticBorders(cc, &top_static_border_size, - &bottom_static_border_size)); - - // Decides if solid background color padding is possible and sets up color - // interpolation functions in CIELAB. Uses linear interpolation by default. - MP_RETURN_IF_ERROR(FindSolidBackgroundColor( - static_features_, static_features_timestamps_, - options_.solid_background_frames_padding_fraction(), - &has_solid_background_, &background_color_l_function_, - &background_color_a_function_, &background_color_b_function_)); - - // Computes key frame crop regions and moves information from raw - // key_frame_infos_ to key_frame_crop_results. - MP_RETURN_IF_ERROR(InitializeFrameCropRegionComputer()); - const int num_key_frames = key_frame_infos_.size(); - std::vector key_frame_crop_results(num_key_frames); - for (int i = 0; i < num_key_frames; ++i) { - MP_RETURN_IF_ERROR(frame_crop_region_computer_->ComputeFrameCropRegion( - key_frame_infos_[i], &key_frame_crop_results[i])); - } - - SceneKeyFrameCropSummary scene_summary; - std::vector focus_point_frames; - SceneCameraMotion scene_camera_motion; - MP_RETURN_IF_ERROR( - scene_camera_motion_analyzer_->AnalyzeSceneAndPopulateFocusPointFrames( - key_frame_crop_options_, key_frame_crop_results, frame_width_, - effective_frame_height_, scene_frame_timestamps_, - has_solid_background_, &scene_summary, &focus_point_frames, - &scene_camera_motion)); - - // Crops scene frames. - std::vector cropped_frames; - std::vector crop_from_locations; - - auto* cropped_frames_ptr = - should_perform_frame_cropping_ ? &cropped_frames : nullptr; - - MP_RETURN_IF_ERROR(scene_cropper_->CropFrames( - scene_summary, scene_frame_timestamps_, is_key_frames_, - scene_frames_or_empty_, focus_point_frames, prior_focus_point_frames_, - top_static_border_size, bottom_static_border_size, continue_last_scene_, - &crop_from_locations, cropped_frames_ptr)); - - // Formats and outputs cropped frames. - bool apply_padding = false; - float vertical_fill_percent; - std::vector render_to_locations; - std::vector padding_colors; - MP_RETURN_IF_ERROR(FormatAndOutputCroppedFrames( - scene_summary.crop_window_width(), scene_summary.crop_window_height(), - scene_frame_timestamps_.size(), &render_to_locations, &apply_padding, - &padding_colors, &vertical_fill_percent, cropped_frames_ptr, cc)); - // Caches prior FocusPointFrames if this was not the end of a scene. - prior_focus_point_frames_.clear(); - if (!is_end_of_scene) { - const int start = - std::max(0, static_cast(scene_frame_timestamps_.size()) - - options_.camera_motion_options() - .polynomial_path_solver() - .prior_frame_buffer_size()); - for (int i = start; i < num_key_frames; ++i) { - prior_focus_point_frames_.push_back(focus_point_frames[i]); - } - } - - // Optionally outputs visualization frames. - MP_RETURN_IF_ERROR(OutputVizFrames(key_frame_crop_results, focus_point_frames, - crop_from_locations, - scene_summary.crop_window_width(), - scene_summary.crop_window_height(), cc)); - - const double start_sec = Timestamp(scene_frame_timestamps_.front()).Seconds(); - const double end_sec = Timestamp(scene_frame_timestamps_.back()).Seconds(); - VLOG(1) << absl::StrFormat("Processed a scene from %.2f sec to %.2f sec", - start_sec, end_sec); - - // Optionally makes summary. - if (cc->Outputs().HasTag(kOutputSummary)) { - auto* scene_summary = summary_->add_scene_summaries(); - scene_summary->set_start_sec(start_sec); - scene_summary->set_end_sec(end_sec); - *(scene_summary->mutable_camera_motion()) = scene_camera_motion; - scene_summary->set_is_end_of_scene(is_end_of_scene); - scene_summary->set_is_padded(apply_padding); - } - - if (cc->Outputs().HasTag(kExternalRenderingPerFrame)) { - for (int i = 0; i < scene_frame_timestamps_.size(); i++) { - auto external_render_message = absl::make_unique(); - ConstructExternalRenderMessage( - crop_from_locations[i], render_to_locations[i], padding_colors[i], - scene_frame_timestamps_[i], external_render_message.get()); - cc->Outputs() - .Tag(kExternalRenderingPerFrame) - .Add(external_render_message.release(), - Timestamp(scene_frame_timestamps_[i])); - } - } - - if (cc->Outputs().HasTag(kExternalRenderingFullVid)) { - for (int i = 0; i < scene_frame_timestamps_.size(); i++) { - ExternalRenderFrame render_frame; - ConstructExternalRenderMessage(crop_from_locations[i], - render_to_locations[i], padding_colors[i], - scene_frame_timestamps_[i], &render_frame); - external_render_list_->push_back(render_frame); - } - } - - key_frame_infos_.clear(); - scene_frames_or_empty_.clear(); - scene_frame_timestamps_.clear(); - is_key_frames_.clear(); - static_features_.clear(); - static_features_timestamps_.clear(); - return absl::OkStatus(); -} - -absl::Status SceneCroppingCalculator::FormatAndOutputCroppedFrames( - const int crop_width, const int crop_height, const int num_frames, - std::vector* render_to_locations, bool* apply_padding, - std::vector* padding_colors, float* vertical_fill_percent, - const std::vector* cropped_frames_ptr, CalculatorContext* cc) { - RET_CHECK(apply_padding) << "Has padding boolean is null."; - - // Computes scaling factor and decides if padding is needed. - VLOG(1) << "crop_width = " << crop_width << " crop_height = " << crop_height; - const double scaling = - std::max(static_cast(target_width_) / crop_width, - static_cast(target_height_) / crop_height); - int scaled_width = std::round(scaling * crop_width); - int scaled_height = std::round(scaling * crop_height); - RET_CHECK_GE(scaled_width, target_width_) - << "Scaled width is less than target width - something is wrong."; - RET_CHECK_GE(scaled_height, target_height_) - << "Scaled height is less than target height - something is wrong."; - if (scaled_width - target_width_ <= 1) scaled_width = target_width_; - if (scaled_height - target_height_ <= 1) scaled_height = target_height_; - *apply_padding = - scaled_width != target_width_ || scaled_height != target_height_; - *vertical_fill_percent = scaled_height / static_cast(target_height_); - if (*apply_padding) { - padder_ = absl::make_unique( - scaled_width, scaled_height, target_aspect_ratio_); - VLOG(1) << "Scene is padded: scaled width = " << scaled_width - << " target width = " << target_width_ - << " scaled height = " << scaled_height - << " target height = " << target_height_; - } - - // Compute the "render to" location. This is where the rect taken from the - // input video gets pasted on the output frame. For use with external - // rendering solutions. - for (int i = 0; i < num_frames; i++) { - if (*apply_padding) { - render_to_locations->push_back(padder_->ComputeOutputLocation()); - } else { - render_to_locations->push_back( - cv::Rect(0, 0, target_width_, target_height_)); - } - } - - // Compute padding colors. - for (int i = 0; i < num_frames; ++i) { - // Set default padding color to white. - cv::Scalar padding_color_to_add = cv::Scalar(255, 255, 255); - const int64 time_ms = scene_frame_timestamps_[i]; - if (*apply_padding) { - if (has_solid_background_) { - double lab[3]; - lab[0] = background_color_l_function_.Evaluate(time_ms); - lab[1] = background_color_a_function_.Evaluate(time_ms); - lab[2] = background_color_b_function_.Evaluate(time_ms); - cv::Mat3f lab_mat(1, 1, cv::Vec3f(lab[0], lab[1], lab[2])); - cv::Mat3f rgb_mat(1, 1); - // Necessary scaling of the RGB values from [0, 1] to [0, 255] based on: - // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor - cv::cvtColor(lab_mat, rgb_mat, cv::COLOR_Lab2RGB); - rgb_mat *= 255.0; - auto k = rgb_mat.at(0, 0); - k[0] = k[0] < 0.0 ? 0.0 : k[0] > 255.0 ? 255.0 : k[0]; - k[1] = k[1] < 0.0 ? 0.0 : k[1] > 255.0 ? 255.0 : k[1]; - k[2] = k[2] < 0.0 ? 0.0 : k[2] > 255.0 ? 255.0 : k[2]; - cv::Scalar interpolated_color = - cv::Scalar(std::round(k[0]), std::round(k[1]), std::round(k[2])); - padding_color_to_add = interpolated_color; - } - } - padding_colors->push_back(padding_color_to_add); - } - if (!cropped_frames_ptr) { - return absl::OkStatus(); - } - - // Resizes cropped frames, pads frames, and output frames. - for (int i = 0; i < num_frames; ++i) { - const int64 time_ms = scene_frame_timestamps_[i]; - const Timestamp timestamp(time_ms); - auto scaled_frame = absl::make_unique( - frame_format_, scaled_width, scaled_height); - auto destination = formats::MatView(scaled_frame.get()); - if (scaled_width == crop_width && scaled_height == crop_height) { - cropped_frames_ptr->at(i).copyTo(destination); - } else { - // cubic is better quality for upscaling and area is good for - // downscaling - const int interpolation_method = - scaling > 1 ? cv::INTER_CUBIC : cv::INTER_AREA; - cv::resize(cropped_frames_ptr->at(i), destination, destination.size(), 0, - 0, interpolation_method); - } - if (*apply_padding) { - cv::Scalar* background_color = nullptr; - if (has_solid_background_) { - background_color = &padding_colors->at(i); - } - auto padded_frame = absl::make_unique(); - MP_RETURN_IF_ERROR(padder_->Process( - *scaled_frame, background_contrast_, - std::min({blur_cv_size_, scaled_width, scaled_height}), - overlay_opacity_, padded_frame.get(), background_color)); - RET_CHECK_EQ(padded_frame->Width(), target_width_) - << "Padded frame width is off."; - RET_CHECK_EQ(padded_frame->Height(), target_height_) - << "Padded frame height is off."; - cc->Outputs() - .Tag(kOutputCroppedFrames) - .Add(padded_frame.release(), timestamp); - } else { - cc->Outputs() - .Tag(kOutputCroppedFrames) - .Add(scaled_frame.release(), timestamp); - } - } - return absl::OkStatus(); -} - -absl::Status SceneCroppingCalculator::OutputVizFrames( - const std::vector& key_frame_crop_results, - const std::vector& focus_point_frames, - const std::vector& crop_from_locations, - const int crop_window_width, const int crop_window_height, - CalculatorContext* cc) const { - if (cc->Outputs().HasTag(kOutputKeyFrameCropViz)) { - std::vector> viz_frames; - MP_RETURN_IF_ERROR(DrawDetectionsAndCropRegions( - scene_frames_or_empty_, is_key_frames_, key_frame_infos_, - key_frame_crop_results, frame_format_, &viz_frames)); - for (int i = 0; i < scene_frames_or_empty_.size(); ++i) { - cc->Outputs() - .Tag(kOutputKeyFrameCropViz) - .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i])); - } - } - if (cc->Outputs().HasTag(kOutputFocusPointFrameViz)) { - std::vector> viz_frames; - MP_RETURN_IF_ERROR(DrawFocusPointAndCropWindow( - scene_frames_or_empty_, focus_point_frames, - options_.viz_overlay_opacity(), crop_window_width, crop_window_height, - frame_format_, &viz_frames)); - for (int i = 0; i < scene_frames_or_empty_.size(); ++i) { - cc->Outputs() - .Tag(kOutputFocusPointFrameViz) - .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i])); - } - } - if (cc->Outputs().HasTag(kOutputFramingAndDetections)) { - std::vector> viz_frames; - MP_RETURN_IF_ERROR(DrawDetectionAndFramingWindow( - raw_scene_frames_or_empty_, crop_from_locations, frame_format_, - options_.viz_overlay_opacity(), &viz_frames)); - for (int i = 0; i < raw_scene_frames_or_empty_.size(); ++i) { - cc->Outputs() - .Tag(kOutputFramingAndDetections) - .Add(viz_frames[i].release(), Timestamp(scene_frame_timestamps_[i])); - } - } - return absl::OkStatus(); -} - -REGISTER_CALCULATOR(SceneCroppingCalculator); - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h deleted file mode 100644 index 61b7b53d6..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_ - -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h" -#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h" -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" -#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h" -#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h" -#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { -// This calculator crops video scenes to target size, which can be of any aspect -// ratio. The calculator supports both "landscape -> portrait", and "portrait -> -// landscape" use cases. The two use cases are automatically determined by -// comparing the input and output frame's aspect ratios internally. -// -// The target (i.e. output) frame's dimension can be specified through the -// target_width(height) fields in the options. Both this target dimension and -// the input dimension should be even. If either keep_original_height or -// keep_original_width is set to true, the corresponding target dimension will -// only be used to compute the aspect ratio (as opposed to setting the actual -// dimension) of the output. If the output frame thus computed has an odd -// size, it will be rounded down to an even number. -// -// The calculator takes shot boundary signals to identify shot boundaries, and -// crops each scene independently. The cropping decisions are made based on -// detection features, which are a collection of focus regions detected from -// different signals, and then fused together by a SignalFusingCalculator. To -// add a new type of focus signals, it should be added in the input of the -// SignalFusingCalculator, which can take an arbitrary number of input streams. -// -// If after attempting to cover focus regions based on the cropping decisions -// made, the retained frame region's aspect ratio is still different from the -// target aspect ratio, padding will be applied. In this case, a seamless -// padding with a solid color would be preferred wherever possible, given -// information from the input static features; otherwise, a simple padding with -// centered foreground on blurred background will be applied. -// -// The main complexity of this calculator lies in stabilizing crop regions over -// the scene using a Retargeter, which solves linear programming problems -// through a L1 path solver (default) or least squares problems through a L2 -// path solver. - -// Input streams: -// - required tag VIDEO_FRAMES (type ImageFrame): -// Original scene frames to be cropped. -// - required tag DETECTION_FEATURES (type DetectionSet): -// Detected features on the key frames. -// - optional tag STATIC_FEATURES (type StaticFeatures): -// Detected features on the key frames. -// - required tag SHOT_BOUNDARIES (type bool): -// Indicators for shot boundaries (output of shot boundary detection). -// - optional tag KEY_FRAMES (type ImageFrame): -// Key frames on which features are detected. This is only used to set the -// detection features frame size. Alternatively, set -// video_feature_width/video_features_height within the options proto to -// define this value. When neither is set, the features frame size is -// assumed to be the original scene frame size. -// -// Output streams: -// - required tag CROPPED_FRAMES (type ImageFrame): -// Cropped frames at target size and original frame rate. -// - optional tag KEY_FRAME_CROP_REGION_VIZ_FRAMES (type ImageFrame): -// Debug visualization frames at original frame size and frame rate. Draws -// the required (yellow) and non-required (cyan) detection features and the -// key frame crop regions (green). -// - optional tag SALIENT_POINT_FRAME_VIZ_FRAMES (type ImageFrame): -// Debug visualization frames at original frame size and frame rate. Draws -// the focus points and the scene crop window (red). -// - optional tag CROPPING_SUMMARY (type VideoCroppingSummary): -// Debug summary information for the video. Only generates one packet when -// calculator closes. -// - optional tag EXTERNAL_RENDERING_PER_FRAME (type ExternalRenderFrame) -// Provides a per-frame message that can be used to render autoflip using an -// external renderer. -// - optional tag EXTERNAL_RENDERING_FULL_VID (type Vector) -// Provides an end-stream message that can be used to render autoflip using -// an external renderer. -// -// Example config: -// node { -// calculator: "SceneCroppingCalculator" -// input_stream: "VIDEO_FRAMES:camera_frames_org" -// input_stream: "KEY_FRAMES:down_sampled_frames" -// input_stream: "DETECTION_FEATURES:focus_regions" -// input_stream: "STATIC_FEATURES:border_features" -// input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" -// output_stream: "CROPPED_FRAMES:cropped_frames" -// options: { -// [mediapipe.SceneCroppingCalculatorOptions.ext]: { -// target_width: 720 -// target_height: 1124 -// target_size_type: USE_TARGET_DIMENSION -// } -// } -// } -// Note that only the target size is required in the options, and all other -// fields are optional with default settings. -class SceneCroppingCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc); - - // Validates calculator options and initializes SceneCameraMotionAnalyzer and - // SceneCropper. - absl::Status Open(CalculatorContext* cc) override; - - // Buffers each scene frame and its timestamp. Packs and stores KeyFrameInfo - // for key frames (a.k.a. frames with detection features). When a shot - // boundary is encountered or when the buffer is full, calls ProcessScene() - // to process the scene at once, and clears buffers. - absl::Status Process(CalculatorContext* cc) override; - - // Calls ProcessScene() on remaining buffered frames. Optionally outputs a - // VideoCroppingSummary if the output stream CROPPING_SUMMARY is present. - absl::Status Close(mediapipe::CalculatorContext* cc) override; - - private: - // Removes any static borders from the scene frames before cropping. The - // arguments |top_border_size| and |bottom_border_size| report the size of the - // removed borders. - absl::Status RemoveStaticBorders(CalculatorContext* cc, int* top_border_size, - int* bottom_border_size); - - // Sets up autoflip after first frame is received and input size is known. - absl::Status InitializeSceneCroppingCalculator( - mediapipe::CalculatorContext* cc); - // Initializes a FrameCropRegionComputer given input and target frame sizes. - absl::Status InitializeFrameCropRegionComputer(); - - // Processes a scene using buffered scene frames and KeyFrameInfos: - // 1. Computes key frame crop regions using a FrameCropRegionComputer. - // 2. Analyzes scene camera motion and generates FocusPointFrames using a - // SceneCameraMotionAnalyzer. - // 3. Crops scene frames using a SceneCropper (wrapper around Retargeter). - // 4. Formats and outputs cropped frames . - // 5. Caches prior FocusPointFrames if this is not the end of a scene (due - // to force flush). - // 6. Optionally outputs visualization frames. - // 7. Optionally updates cropping summary. - absl::Status ProcessScene(const bool is_end_of_scene, CalculatorContext* cc); - - // Formats and outputs the cropped frames passed in through - // |cropped_frames_ptr|. Scales them to be at least as big as the target - // size. If the aspect ratio is different, applies padding. Uses solid - // background from static features if possible, otherwise uses blurred - // background. Sets |apply_padding| to true if the scene is padded. Set - // |cropped_frames_ptr| to nullptr, to bypass the actual output of the - // cropped frames. This is useful when the calculator is only used for - // computing the cropping metadata rather than doing the actual cropping - // operation. - absl::Status FormatAndOutputCroppedFrames( - const int crop_width, const int crop_height, const int num_frames, - std::vector* render_to_locations, bool* apply_padding, - std::vector* padding_colors, float* vertical_fill_percent, - const std::vector* cropped_frames_ptr, CalculatorContext* cc); - - // Draws and outputs visualization frames if those streams are present. - absl::Status OutputVizFrames( - const std::vector& key_frame_crop_results, - const std::vector& focus_point_frames, - const std::vector& crop_from_locations, - const int crop_window_width, const int crop_window_height, - CalculatorContext* cc) const; - - // Filters detections based on USER_HINT under specific flag conditions. - void FilterKeyFrameInfo(); - - // Target frame size and aspect ratio passed in or computed from options. - int target_width_ = -1; - int target_height_ = -1; - double target_aspect_ratio_ = -1.0; - - // Input video frame size and format. - int frame_width_ = -1; - int frame_height_ = -1; - ImageFormat::Format frame_format_ = ImageFormat::UNKNOWN; - - // Key frame size (frame size for detections and border detections). - int key_frame_width_ = -1; - int key_frame_height_ = -1; - - // Calculator options. - SceneCroppingCalculatorOptions options_; - - // Buffered KeyFrameInfos for the current scene (size = number of key - // frames). - std::vector key_frame_infos_; - - // Buffered frames, timestamps, and indicators for key frames in the current - // scene (size = number of input video frames). - // Note: scene_frames_or_empty_ may be empty if the actual cropping - // operation of frames is turned off, e.g. when - // |should_perform_frame_cropping_| is false, so rely on - // scene_frame_timestamps_.size() to query the number of accumulated - // timestamps rather than scene_frames_or_empty_.size(). - // TODO: all of the following vectors are expected to be the same - // size. Add to struct and store together in one vector. - std::vector scene_frames_or_empty_; - std::vector raw_scene_frames_or_empty_; - std::vector scene_frame_timestamps_; - std::vector is_key_frames_; - - // Static border information for the scene. - int top_border_distance_ = -1; - int effective_frame_height_ = -1; - - // Stored FocusPointFrames from prior scene when there was no actual scene - // change (due to forced flush when buffer is full). - std::vector prior_focus_point_frames_; - // Indicates if this scene is a continuation of the last scene (due to - // forced flush when buffer is full). - bool continue_last_scene_ = false; - - // KeyFrameCropOptions used by the FrameCropRegionComputer. - KeyFrameCropOptions key_frame_crop_options_; - - // Object for computing key frame crop regions from detection features. - std::unique_ptr frame_crop_region_computer_ = - nullptr; - - // Object for analyzing scene camera motion from key frame crop regions and - // generating FocusPointFrames. - std::unique_ptr scene_camera_motion_analyzer_ = - nullptr; - - // Object for cropping a scene given FocusPointFrames. - std::unique_ptr scene_cropper_ = nullptr; - - // Buffered static features and their timestamps used in padding with solid - // background color (size = number of frames with static features). - std::vector static_features_; - std::vector static_features_timestamps_; - bool has_solid_background_ = false; - // CIELAB yields more natural color transitions than RGB and HSV: RGB tends - // to produce darker in-between colors and HSV can introduce new hues. See - // https://howaboutanorange.com/blog/2011/08/10/color_interpolation/ for - // visual comparisons of color transition in different spaces. - PiecewiseLinearFunction background_color_l_function_; // CIELAB - l - PiecewiseLinearFunction background_color_a_function_; // CIELAB - a - PiecewiseLinearFunction background_color_b_function_; // CIELAB - b - - // Parameters for padding with blurred background passed in from options. - float background_contrast_ = -1.0; - int blur_cv_size_ = -1; - float overlay_opacity_ = -1.0; - // Object for padding an image to a target aspect ratio. - std::unique_ptr padder_ = nullptr; - - // Optional diagnostic summary output emitted in Close(). - std::unique_ptr summary_ = nullptr; - - // Optional list of external rendering messages for each processed frame. - std::unique_ptr> external_render_list_; - - // Determines whether to perform real cropping on input frames. This flag is - // useful when the user only needs to compute cropping windows, in which - // case setting this flag to false can avoid buffering as well as cropping - // frames. This can significantly reduce memory usage and speed up - // processing. Some debugging visualization inevitably will be disabled - // because of this flag too. - bool should_perform_frame_cropping_ = false; -}; -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_CALCULATORS_SCENE_CROPPING_CALCULATOR_H_ diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto deleted file mode 100644 index f9ba9cb87..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.proto +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/examples/desktop/autoflip/quality/cropping.proto"; -import "mediapipe/framework/calculator.proto"; - -// Options for the SceneCroppingCalculator. -message SceneCroppingCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional SceneCroppingCalculatorOptions ext = 284806831; - } - - // Target frame size - this has to be even (for ffmpeg encoding). - optional int32 target_width = 1; - optional int32 target_height = 2; - - // Choices for target size specification. - enum TargetSizeType { - // Unknown type (needed by ProtoBestPractices to ensure consistent behavior - // across proto2 and proto3). This type should not be used. - UNKNOWN = 0; - // Directly uses the target dimension given above. - USE_TARGET_DIMENSION = 1; - // Uses the target dimension to compute the target aspect ratio, but keeps - // original height/width. If the resulting size for the other dimension is - // odd, it is rounded down to an even size. - KEEP_ORIGINAL_HEIGHT = 2; - KEEP_ORIGINAL_WIDTH = 3; - // Used on conjuntion with external_aspect_ratio, create the largest sized - // output without upscaling the video. - MAXIMIZE_TARGET_DIMENSION = 4; - // Uses original dimensions to calculate aspect ratio. - KEEP_ORIGINAL_DIMENSION = 5; - } - optional TargetSizeType target_size_type = 3 [default = USE_TARGET_DIMENSION]; - - // Forces a flush of the frame buffer after this number of frames even if - // there is not a shot boundary. - optional int32 max_scene_size = 4 [default = 600]; - - // Number of frames from prior buffer to be used to smooth out camera - // trajectory when it was a forced flush. - optional int32 prior_frame_buffer_size = 5 [default = 30, deprecated = true]; - // Set camera motion type along with parameters. Must select between the two - // provided options. - optional CameraMotionOptions camera_motion_options = 14; - - // Options for computing key frame crop regions using the - // FrameCropRegionComputer. - // **** Note: You shall NOT manually set the target width and height fields - // inside this field as they will be overridden internally in the calculator - // (i.e. automatically computed from target aspect ratio). - optional KeyFrameCropOptions key_frame_crop_options = 6; - - // Options for analyzing scene camera motion and populating SalientPointFrames - // using the SceneCameraMotionAnalyzer. - optional SceneCameraMotionAnalyzerOptions - scene_camera_motion_analyzer_options = 7; - - // If the fraction of frames with solid background in one shot exceeds this - // threshold, use a solid color for background in padding for this shot. - optional float solid_background_frames_padding_fraction = 8 [default = 0.6]; - - // Options for padding using the PaddingEffectGenerator (copied from - // ad_creation/calculators/universal_padding_calculator.proto). - message PaddingEffectParameters { - // Contrast adjustment for padding background. This value should between 0 - // and 1. The smaller the value, the darker the background. 1 means no - // contrast change. - optional float background_contrast = 1 [default = 1.0]; - // The cv::Size() parameter used in creating blurry effects for padding - // backgrounds. - optional int32 blur_cv_size = 2 [default = 200]; - // The opacity of the black layer overlaied on top of the background. The - // value should be within [0, 1], in which 0 means totally transparent, and - // 1 means totally opaque. - optional float overlay_opacity = 3 [default = 0.6]; - } - optional PaddingEffectParameters padding_parameters = 9; - - // If set and input "KEY_FRAMES" not provided, uses these keyframe values. - optional int32 video_features_width = 10; - optional int32 video_features_height = 11; - - // If a user hint is provided on a scene, use only this signal for cropping - // and camera motion. - optional bool user_hint_override = 12; - - // An opacity used to render cropping windows for visualization purposes. - optional float viz_overlay_opacity = 13 [default = 0.7]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc deleted file mode 100644 index 43f60470e..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc +++ /dev/null @@ -1,908 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.h" - -#include -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -using ::testing::HasSubstr; - -constexpr char kConfig[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_FRAMES:camera_frames_org" - input_stream: "KEY_FRAMES:down_sampled_frames" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "CROPPED_FRAMES:cropped_frames" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - target_size_type: $2 - max_scene_size: $3 - prior_frame_buffer_size: $4 - } - })"; - -constexpr char kNoKeyFrameConfig[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_FRAMES:camera_frames_org" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "CROPPED_FRAMES:cropped_frames" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - } - })"; - -constexpr char kDebugConfigNoCroppedFrame[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_FRAMES:camera_frames_org" - input_stream: "KEY_FRAMES:down_sampled_frames" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames" - output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - } - })"; - -constexpr char kDebugConfig[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_FRAMES:camera_frames_org" - input_stream: "KEY_FRAMES:down_sampled_frames" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "CROPPED_FRAMES:cropped_frames" - output_stream: "KEY_FRAME_CROP_REGION_VIZ_FRAMES:key_frame_crop_viz_frames" - output_stream: "SALIENT_POINT_FRAME_VIZ_FRAMES:salient_point_viz_frames" - output_stream: "FRAMING_DETECTIONS_VIZ_FRAMES:framing_viz_frames" - output_stream: "CROPPING_SUMMARY:cropping_summaries" - output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame" - output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - } - })"; - -constexpr char kExternalRenderConfig[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_FRAMES:camera_frames_org" - input_stream: "KEY_FRAMES:down_sampled_frames" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame" - output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - } - })"; - -constexpr char kExternalRenderConfigNoVideo[] = R"( - calculator: "SceneCroppingCalculator" - input_stream: "VIDEO_SIZE:camera_size" - input_stream: "DETECTION_FEATURES:salient_regions" - input_stream: "STATIC_FEATURES:border_features" - input_stream: "SHOT_BOUNDARIES:shot_boundary_frames" - output_stream: "EXTERNAL_RENDERING_PER_FRAME:external_rendering_per_frame" - output_stream: "EXTERNAL_RENDERING_FULL_VID:external_rendering_full_vid" - options: { - [mediapipe.autoflip.SceneCroppingCalculatorOptions.ext]: { - target_width: $0 - target_height: $1 - video_features_width: $2 - video_features_height: $3 - } - })"; - -constexpr int kInputFrameWidth = 1280; -constexpr int kInputFrameHeight = 720; - -constexpr int kKeyFrameWidth = 640; -constexpr int kKeyFrameHeight = 360; - -constexpr int kTargetWidth = 720; -constexpr int kTargetHeight = 1124; -constexpr SceneCroppingCalculatorOptions::TargetSizeType kTargetSizeType = - SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION; - -constexpr int kNumScenes = 3; -constexpr int kSceneSize = 8; -constexpr int kMaxSceneSize = 10; -constexpr int kPriorFrameBufferSize = 5; - -constexpr int kMinNumDetections = 0; -constexpr int kMaxNumDetections = 10; - -constexpr int kDownSampleRate = 4; -constexpr int64 kTimestampDiff = 20000; - -// Returns a singleton random engine for generating random values. The seed is -// fixed for reproducibility. -std::default_random_engine& GetGen() { - static std::default_random_engine generator{0}; - return generator; -} - -// Returns random color with r, g, b in the range of [0, 255]. -cv::Scalar GetRandomColor() { - std::uniform_int_distribution distribution(0, 255); - const int red = distribution(GetGen()); - const int green = distribution(GetGen()); - const int blue = distribution(GetGen()); - return cv::Scalar(red, green, blue); -} - -// Makes a detection set given number of detections. Each detection has randomly -// generated regions within given width and height with random score in [0, 1], -// and is randomly set to be required or non-required. -std::unique_ptr MakeDetections(const int num_detections, - const int width, - const int height) { - std::uniform_int_distribution width_distribution(0, width); - std::uniform_int_distribution height_distribution(0, height); - std::uniform_real_distribution score_distribution(0.0, 1.0); - std::bernoulli_distribution is_required_distribution(0.5); - auto detections = absl::make_unique(); - for (int i = 0; i < num_detections; ++i) { - auto* region = detections->add_detections(); - const int x1 = width_distribution(GetGen()); - const int x2 = width_distribution(GetGen()); - const int y1 = height_distribution(GetGen()); - const int y2 = height_distribution(GetGen()); - const int x_min = std::min(x1, x2), x_max = std::max(x1, x2); - const int y_min = std::min(y1, y2), y_max = std::max(y1, y2); - auto* location = region->mutable_location(); - location->set_x(x_min); - location->set_width(x_max - x_min); - location->set_y(y_min); - location->set_height(y_max - y_min); - region->set_score(score_distribution(GetGen())); - region->set_is_required(is_required_distribution(GetGen())); - } - return detections; -} - -// Makes a detection set given number of detections. Each detection has randomly -// generated regions within given width and height with random score in [0, 1], -// and is randomly set to be required or non-required. -std::unique_ptr MakeCenterDetection(const int width, - const int height) { - auto detections = absl::make_unique(); - auto* region = detections->add_detections(); - auto* location = region->mutable_location(); - location->set_x(width / 2 - 5); - location->set_width(width / 2 + 10); - location->set_y(height / 2 - 5); - location->set_height(height); - region->set_score(1); - return detections; -} - -// Makes an image frame of solid color given color, width, and height. -std::unique_ptr MakeImageFrameFromColor(const cv::Scalar& color, - const int width, - const int height) { - auto image_frame = - absl::make_unique(ImageFormat::SRGB, width, height); - auto mat = formats::MatView(image_frame.get()); - mat = color; - return image_frame; -} - -// Adds key frame detection features given time (in ms) to the input stream. -// Randomly generates a number of detections in the range of kMinNumDetections -// and kMaxNumDetections. Optionally add a key image frame of random solid color -// and given size. -void AddKeyFrameFeatures(const int64 time_ms, const int key_frame_width, - const int key_frame_height, bool randomize, - CalculatorRunner::StreamContentsSet* inputs) { - Timestamp timestamp(time_ms); - if (inputs->HasTag("KEY_FRAMES")) { - auto key_frame = MakeImageFrameFromColor(GetRandomColor(), key_frame_width, - key_frame_height); - inputs->Tag("KEY_FRAMES") - .packets.push_back(Adopt(key_frame.release()).At(timestamp)); - } - if (randomize) { - const int num_detections = std::uniform_int_distribution( - kMinNumDetections, kMaxNumDetections)(GetGen()); - auto detections = - MakeDetections(num_detections, key_frame_width, key_frame_height); - inputs->Tag("DETECTION_FEATURES") - .packets.push_back(Adopt(detections.release()).At(timestamp)); - } else { - auto detections = MakeCenterDetection(key_frame_width, key_frame_height); - inputs->Tag("DETECTION_FEATURES") - .packets.push_back(Adopt(detections.release()).At(timestamp)); - } -} - -// Adds a scene given number of frames to the input stream. Spaces frame at the -// default timestamp interval starting from given start frame index. Scene has -// empty static features. -void AddScene(const int start_frame_index, const int num_scene_frames, - const int frame_width, const int frame_height, - const int key_frame_width, const int key_frame_height, - const int DownSampleRate, - CalculatorRunner::StreamContentsSet* inputs) { - int64 time_ms = start_frame_index * kTimestampDiff; - for (int i = 0; i < num_scene_frames; ++i) { - Timestamp timestamp(time_ms); - if (inputs->HasTag("VIDEO_FRAMES")) { - auto frame = - MakeImageFrameFromColor(GetRandomColor(), frame_width, frame_height); - inputs->Tag("VIDEO_FRAMES") - .packets.push_back(Adopt(frame.release()).At(timestamp)); - } else { - auto input_size = - ::absl::make_unique>(frame_width, frame_height); - inputs->Tag("VIDEO_SIZE") - .packets.push_back(Adopt(input_size.release()).At(timestamp)); - } - auto static_features = absl::make_unique(); - inputs->Tag("STATIC_FEATURES") - .packets.push_back(Adopt(static_features.release()).At(timestamp)); - if (DownSampleRate == 1) { - AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, false, - inputs); - } else if (i % DownSampleRate == 0) { // is a key frame - AddKeyFrameFeatures(time_ms, key_frame_width, key_frame_height, true, - inputs); - } - if (i == num_scene_frames - 1) { // adds shot boundary - inputs->Tag("SHOT_BOUNDARIES") - .packets.push_back(Adopt(new bool(true)).At(Timestamp(time_ms))); - } - time_ms += kTimestampDiff; - } -} - -// Checks that the output stream for cropped frames has the correct number of -// frames, and that the size of each frame is correct. -void CheckCroppedFrames(const CalculatorRunner& runner, const int num_frames, - const int target_width, const int target_height) { - const auto& outputs = runner.Outputs(); - EXPECT_TRUE(outputs.HasTag("CROPPED_FRAMES")); - const auto& cropped_frames_outputs = outputs.Tag("CROPPED_FRAMES").packets; - EXPECT_EQ(cropped_frames_outputs.size(), num_frames); - for (int i = 0; i < num_frames; ++i) { - const auto& cropped_frame = cropped_frames_outputs[i].Get(); - EXPECT_EQ(cropped_frame.Width(), target_width); - EXPECT_EQ(cropped_frame.Height(), target_height); - } -} - -// Checks that the calculator checks the maximum scene size is valid. -TEST(SceneCroppingCalculatorTest, ChecksMaxSceneSize) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kConfig, kTargetWidth, kTargetHeight, - kTargetSizeType, 0, kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - const auto status = runner->Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Maximum scene size is non-positive.")); -} - -// Checks that the calculator checks the prior frame buffer size is valid. -TEST(SceneCroppingCalculatorTest, ChecksPriorFrameBufferSize) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kConfig, kTargetWidth, kTargetHeight, - kTargetSizeType, kMaxSceneSize, -1)); - auto runner = absl::make_unique(config); - const auto status = runner->Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Prior frame buffer size is negative.")); -} - -TEST(SceneCroppingCalculatorTest, ChecksDebugConfigWithoutCroppedFrame) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kDebugConfigNoCroppedFrame, kTargetWidth, kTargetHeight)); - auto runner = absl::make_unique(config); - const auto status = runner->Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("can only be used when")); -} - -// Checks that the calculator crops scene frames when there is no input key -// frames stream. -TEST(SceneCroppingCalculatorTest, HandlesNoKeyFrames) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kNoKeyFrameConfig, kTargetWidth, kTargetHeight)); - auto runner = absl::make_unique(config); - AddScene(0, kSceneSize, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, kDownSampleRate, runner->MutableInputs()); - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, kSceneSize, kTargetWidth, kTargetHeight); -} - -// Checks that the calculator handles scenes longer than maximum scene size ( -// force flush is triggered). -TEST(SceneCroppingCalculatorTest, HandlesLongScene) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize, - kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - AddScene(0, 2 * kMaxSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, 2 * kMaxSceneSize, kTargetWidth, kTargetHeight); -} - -// Checks that the calculator can optionally output debug streams. -TEST(SceneCroppingCalculatorTest, OutputsDebugStreams) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kDebugConfig, kTargetWidth, kTargetHeight)); - auto runner = absl::make_unique(config); - const int num_frames = kSceneSize; - AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, kDownSampleRate, runner->MutableInputs()); - - MP_EXPECT_OK(runner->Run()); - const auto& outputs = runner->Outputs(); - EXPECT_TRUE(outputs.HasTag("KEY_FRAME_CROP_REGION_VIZ_FRAMES")); - EXPECT_TRUE(outputs.HasTag("SALIENT_POINT_FRAME_VIZ_FRAMES")); - EXPECT_TRUE(outputs.HasTag("CROPPING_SUMMARY")); - EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_PER_FRAME")); - EXPECT_TRUE(outputs.HasTag("EXTERNAL_RENDERING_FULL_VID")); - EXPECT_TRUE(outputs.HasTag("FRAMING_DETECTIONS_VIZ_FRAMES")); - const auto& crop_region_viz_frames_outputs = - outputs.Tag("KEY_FRAME_CROP_REGION_VIZ_FRAMES").packets; - const auto& salient_point_viz_frames_outputs = - outputs.Tag("SALIENT_POINT_FRAME_VIZ_FRAMES").packets; - const auto& summary_output = outputs.Tag("CROPPING_SUMMARY").packets; - const auto& ext_render_per_frame = - outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets; - const auto& ext_render_full_vid = - outputs.Tag("EXTERNAL_RENDERING_FULL_VID").packets; - const auto& framing_viz_frames_output = - outputs.Tag("FRAMING_DETECTIONS_VIZ_FRAMES").packets; - EXPECT_EQ(crop_region_viz_frames_outputs.size(), num_frames); - EXPECT_EQ(salient_point_viz_frames_outputs.size(), num_frames); - EXPECT_EQ(framing_viz_frames_output.size(), num_frames); - EXPECT_EQ(summary_output.size(), 1); - EXPECT_EQ(ext_render_per_frame.size(), num_frames); - EXPECT_EQ(ext_render_full_vid.size(), 1); - EXPECT_EQ(ext_render_per_frame[0].Get().timestamp_us(), - 0); - EXPECT_EQ(ext_render_full_vid[0] - .Get>()[0] - .timestamp_us(), - 0); - EXPECT_EQ(ext_render_per_frame[1].Get().timestamp_us(), - 20000); - EXPECT_EQ(ext_render_full_vid[0] - .Get>()[1] - .timestamp_us(), - 20000); - - for (int i = 0; i < num_frames; ++i) { - const auto& crop_region_viz_frame = - crop_region_viz_frames_outputs[i].Get(); - EXPECT_EQ(crop_region_viz_frame.Width(), kInputFrameWidth); - EXPECT_EQ(crop_region_viz_frame.Height(), kInputFrameHeight); - const auto& salient_point_viz_frame = - salient_point_viz_frames_outputs[i].Get(); - EXPECT_EQ(salient_point_viz_frame.Width(), kInputFrameWidth); - EXPECT_EQ(salient_point_viz_frame.Height(), kInputFrameHeight); - } - const auto& summary = summary_output[0].Get(); - EXPECT_EQ(summary.scene_summaries_size(), 2); - const auto& summary_0 = summary.scene_summaries(0); - EXPECT_TRUE(summary_0.is_padded()); - EXPECT_TRUE(summary_0.camera_motion().has_steady_motion()); -} - -// Checks that the calculator handles the case of generating landscape frames. -TEST(SceneCroppingCalculatorTest, HandlesLandscapeTarget) { - const int input_width = 900; - const int input_height = 1600; - const int target_width = 1200; - const int target_height = 800; - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, target_width, target_height, kTargetSizeType, kMaxSceneSize, - kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - for (int i = 0; i < kNumScenes; ++i) { - AddScene(i * kSceneSize, kSceneSize, input_width, input_height, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - } - const int num_frames = kSceneSize * kNumScenes; - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, num_frames, target_width, target_height); -} - -// Checks that the calculator crops scene frames to target size when the target -// size type is the default USE_TARGET_DIMENSION. -TEST(SceneCroppingCalculatorTest, CropsToTargetSize) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, kTargetWidth, kTargetHeight, kTargetSizeType, kMaxSceneSize, - kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - for (int i = 0; i < kNumScenes; ++i) { - AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - } - const int num_frames = kSceneSize * kNumScenes; - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, num_frames, kTargetWidth, kTargetHeight); -} - -// Checks that the calculator crops scene frames to input size when the target -// size type is KEEP_ORIGINAL_DIMENSION. -TEST(SceneCroppingCalculatorTest, CropsToOriginalDimension) { - // target_width and target_height are ignored - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, /*target_width*/ 2, /*target_height*/ 2, - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_DIMENSION, - kMaxSceneSize, kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - for (int i = 0; i < kNumScenes; ++i) { - AddScene(i * kSceneSize, kSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - } - const int num_frames = kSceneSize * kNumScenes; - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, num_frames, kInputFrameWidth, kInputFrameHeight); -} - -// Checks that the calculator keeps original height if the target size type is -// set to KEEP_ORIGINAL_HEIGHT. -TEST(SceneCroppingCalculatorTest, KeepsOriginalHeight) { - const auto target_size_type = - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT; - const int target_height = kInputFrameHeight; - const double target_aspect_ratio = - static_cast(kTargetWidth) / kTargetHeight; - int target_width = std::round(target_height * target_aspect_ratio); - if (target_width % 2 == 1) target_width--; - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize, - kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height); -} - -// Checks that the calculator keeps original width if the target size type is -// set to KEEP_ORIGINAL_WIDTH. -TEST(SceneCroppingCalculatorTest, KeepsOriginalWidth) { - const auto target_size_type = - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH; - const int target_width = kInputFrameWidth; - const double target_aspect_ratio = - static_cast(kTargetWidth) / kTargetHeight; - int target_height = std::round(target_width / target_aspect_ratio); - if (target_height % 2 == 1) target_height--; - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, kTargetWidth, kTargetHeight, target_size_type, kMaxSceneSize, - kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - MP_EXPECT_OK(runner->Run()); - CheckCroppedFrames(*runner, kMaxSceneSize, target_width, target_height); -} - -// Checks that the calculator rejects odd target size. -TEST(SceneCroppingCalculatorTest, RejectsOddTargetSize) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, kTargetWidth - 1, kTargetHeight, kTargetSizeType, - kMaxSceneSize, kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - AddScene(0, kMaxSceneSize, kInputFrameWidth, kInputFrameHeight, - kKeyFrameWidth, kKeyFrameHeight, kDownSampleRate, - runner->MutableInputs()); - const auto status = runner->Run(); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Target width cannot be odd")); -} - -// Checks that the calculator always produces even frame size given even input -// frame size and even target under all target size types. -TEST(SceneCroppingCalculatorTest, ProducesEvenFrameSize) { - // Some commonly used video resolution (some are divided by 10 to make the - // test faster), and some odd input frame sizes. - const std::vector> video_sizes = { - {384, 216}, {256, 144}, {192, 108}, {128, 72}, {640, 360}, - {426, 240}, {100, 100}, {214, 100}, {240, 100}, {720, 1124}, - {90, 160}, {641, 360}, {640, 361}, {101, 101}}; - - const std::vector - target_size_types = {SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION, - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT, - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH}; - - // Exhaustive check on each size as input and each size as output for each - // target size type. - for (int i = 0; i < video_sizes.size(); ++i) { - const int frame_width = video_sizes[i].first; - const int frame_height = video_sizes[i].second; - for (int j = 0; j < video_sizes.size(); ++j) { - const int target_width = video_sizes[j].first; - const int target_height = video_sizes[j].second; - if (target_width % 2 == 1 || target_height % 2 == 1) continue; - for (int k = 0; k < target_size_types.size(); ++k) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie(absl::Substitute( - kConfig, target_width, target_height, target_size_types[k], - kMaxSceneSize, kPriorFrameBufferSize)); - auto runner = absl::make_unique(config); - AddScene(0, 1, frame_width, frame_height, kKeyFrameWidth, - kKeyFrameHeight, kDownSampleRate, runner->MutableInputs()); - MP_EXPECT_OK(runner->Run()); - const auto& output_frame = runner->Outputs() - .Tag("CROPPED_FRAMES") - .packets[0] - .Get(); - EXPECT_EQ(output_frame.Width() % 2, 0); - EXPECT_EQ(output_frame.Height() % 2, 0); - if (target_size_types[k] == - SceneCroppingCalculatorOptions::USE_TARGET_DIMENSION) { - EXPECT_EQ(output_frame.Width(), target_width); - EXPECT_EQ(output_frame.Height(), target_height); - } else if (target_size_types[k] == - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_HEIGHT) { - // Difference could be 1 if input size is odd. - EXPECT_LE(std::abs(output_frame.Height() - frame_height), 1); - } else if (target_size_types[k] == - SceneCroppingCalculatorOptions::KEEP_ORIGINAL_WIDTH) { - EXPECT_LE(std::abs(output_frame.Width() - frame_width), 1); - } - } - } - } -} - -// Checks that the calculator pads the frames with solid color when possible. -TEST(SceneCroppingCalculatorTest, PadsWithSolidColorFromStaticFeatures) { - const int target_width = 100, target_height = 200; - const int input_width = 100, input_height = 100; - CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kNoKeyFrameConfig, target_width, target_height)); - auto* options = config.mutable_options()->MutableExtension( - SceneCroppingCalculatorOptions::ext); - options->set_solid_background_frames_padding_fraction(0.6); - auto runner = absl::make_unique(config); - - const int static_features_downsample_rate = 2; - const float fraction_with_solid_background = 0.7; - const int red = 122, green = 167, blue = 250; - const int num_frames_with_solid_background = - std::round(fraction_with_solid_background * kSceneSize / - static_features_downsample_rate); - - // Add inputs. - auto* inputs = runner->MutableInputs(); - int64 time_ms = 0; - int num_static_features = 0; - for (int i = 0; i < kSceneSize; ++i) { - Timestamp timestamp(time_ms); - auto frame = - MakeImageFrameFromColor(GetRandomColor(), input_width, input_height); - inputs->Tag("VIDEO_FRAMES") - .packets.push_back(Adopt(frame.release()).At(timestamp)); - if (i % static_features_downsample_rate == 0) { - auto static_features = absl::make_unique(); - if (num_static_features < num_frames_with_solid_background) { - auto* color = static_features->mutable_solid_background(); - // Uses BGR to mimic input from static features solid background color. - color->set_r(blue); - color->set_g(green); - color->set_b(red); - } - inputs->Tag("STATIC_FEATURES") - .packets.push_back(Adopt(static_features.release()).At(timestamp)); - num_static_features++; - } - if (i % kDownSampleRate == 0) { // is a key frame - // Target crop size is (50, 100). Adds one required detection with size - // (80, 100) larger than the target crop size to force padding. - auto detections = absl::make_unique(); - auto* salient_region = detections->add_detections(); - salient_region->set_is_required(true); - auto* location = salient_region->mutable_location(); - location->set_x(10); - location->set_y(0); - location->set_width(80); - location->set_height(input_height); - inputs->Tag("DETECTION_FEATURES") - .packets.push_back(Adopt(detections.release()).At(timestamp)); - } - time_ms += kTimestampDiff; - } - - MP_EXPECT_OK(runner->Run()); - - // Checks that the top and bottom borders indeed have the background color. - const int border_size = 37; - const auto& cropped_frames_outputs = - runner->Outputs().Tag("CROPPED_FRAMES").packets; - EXPECT_EQ(cropped_frames_outputs.size(), kSceneSize); - for (int i = 0; i < kSceneSize; ++i) { - const auto& cropped_frame = cropped_frames_outputs[i].Get(); - cv::Mat mat = formats::MatView(&cropped_frame); - for (int x = 0; x < target_width; ++x) { - for (int y = 0; y < border_size; ++y) { - EXPECT_EQ(mat.at(y, x)[0], red); - EXPECT_EQ(mat.at(y, x)[1], green); - EXPECT_EQ(mat.at(y, x)[2], blue); - } - for (int y2 = 0; y2 < border_size; ++y2) { - const int y = target_height - 1 - y2; - EXPECT_EQ(mat.at(y, x)[0], red); - EXPECT_EQ(mat.at(y, x)[1], green); - EXPECT_EQ(mat.at(y, x)[2], blue); - } - } - } -} - -// Checks that the calculator removes static borders from frames. -TEST(SceneCroppingCalculatorTest, RemovesStaticBorders) { - const int target_width = 50, target_height = 100; - const int input_width = 100, input_height = 100; - const int top_border_size = 20, bottom_border_size = 20; - const cv::Rect top_border_rect(0, 0, input_width, top_border_size); - const cv::Rect bottom_border_rect(0, input_height - bottom_border_size, - input_width, bottom_border_size); - const cv::Scalar frame_color = cv::Scalar(255, 255, 255); - const cv::Scalar border_color = cv::Scalar(0, 0, 0); - - const auto config = ParseTextProtoOrDie( - absl::Substitute(kNoKeyFrameConfig, target_width, target_height)); - auto runner = absl::make_unique(config); - - // Add inputs. - auto* inputs = runner->MutableInputs(); - const auto timestamp = Timestamp(0); - // Make frame with borders. - auto frame = MakeImageFrameFromColor(frame_color, input_width, input_height); - auto mat = formats::MatView(frame.get()); - mat(top_border_rect) = border_color; - mat(bottom_border_rect) = border_color; - inputs->Tag("VIDEO_FRAMES") - .packets.push_back(Adopt(frame.release()).At(timestamp)); - // Set borders in static features. - auto static_features = absl::make_unique(); - auto* top_part = static_features->add_border(); - top_part->set_relative_position(Border::TOP); - top_part->mutable_border_position()->set_height(top_border_size); - auto* bottom_part = static_features->add_border(); - bottom_part->set_relative_position(Border::BOTTOM); - bottom_part->mutable_border_position()->set_height(bottom_border_size); - inputs->Tag("STATIC_FEATURES") - .packets.push_back(Adopt(static_features.release()).At(timestamp)); - // Add empty detections to ensure no padding is used. - auto detections = absl::make_unique(); - inputs->Tag("DETECTION_FEATURES") - .packets.push_back(Adopt(detections.release()).At(timestamp)); - - MP_EXPECT_OK(runner->Run()); - - // Checks that the top and bottom borders are removed. Each frame should have - // solid color equal to frame color. - const auto& cropped_frames_outputs = - runner->Outputs().Tag("CROPPED_FRAMES").packets; - EXPECT_EQ(cropped_frames_outputs.size(), 1); - const auto& cropped_frame = cropped_frames_outputs[0].Get(); - const auto cropped_mat = formats::MatView(&cropped_frame); - for (int x = 0; x < target_width; ++x) { - for (int y = 0; y < target_height; ++y) { - EXPECT_EQ(cropped_mat.at(y, x)[0], frame_color[0]); - EXPECT_EQ(cropped_mat.at(y, x)[1], frame_color[1]); - EXPECT_EQ(cropped_mat.at(y, x)[2], frame_color[2]); - } - } -} - -// Checks external render message with default poly path solver. -TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPath) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kExternalRenderConfig, kTargetWidth, kTargetHeight)); - auto runner = absl::make_unique(config); - const int num_frames = kSceneSize; - AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, 1, runner->MutableInputs()); - - MP_EXPECT_OK(runner->Run()); - const auto& outputs = runner->Outputs(); - const auto& ext_render_per_frame = - outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets; - EXPECT_EQ(ext_render_per_frame.size(), num_frames); - - for (int i = 0; i < num_frames - 1; ++i) { - const auto& ext_render_message = - ext_render_per_frame[i].Get(); - EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000); - EXPECT_EQ(ext_render_message.crop_from_location().x(), 725); - EXPECT_EQ(ext_render_message.crop_from_location().y(), 0); - EXPECT_EQ(ext_render_message.crop_from_location().width(), 461); - EXPECT_EQ(ext_render_message.crop_from_location().height(), 720); - EXPECT_EQ(ext_render_message.render_to_location().x(), 0); - EXPECT_EQ(ext_render_message.render_to_location().y(), 0); - EXPECT_EQ(ext_render_message.render_to_location().width(), 720); - EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); - } -} - -// Checks external render message with kinematic path solver. -TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPath) { - CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kDebugConfig, kTargetWidth, kTargetHeight)); - auto* options = config.mutable_options()->MutableExtension( - SceneCroppingCalculatorOptions::ext); - auto* kinematic_options = - options->mutable_camera_motion_options()->mutable_kinematic_options(); - kinematic_options->set_min_motion_to_reframe(1.2); - kinematic_options->set_max_velocity(200); - - auto runner = absl::make_unique(config); - const int num_frames = kSceneSize; - AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, 1, runner->MutableInputs()); - - MP_EXPECT_OK(runner->Run()); - const auto& outputs = runner->Outputs(); - const auto& ext_render_per_frame = - outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets; - EXPECT_EQ(ext_render_per_frame.size(), num_frames); - - for (int i = 0; i < num_frames - 1; ++i) { - const auto& ext_render_message = - ext_render_per_frame[i].Get(); - EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000); - EXPECT_EQ(ext_render_message.crop_from_location().x(), 725); - EXPECT_EQ(ext_render_message.crop_from_location().y(), 0); - EXPECT_EQ(ext_render_message.crop_from_location().width(), 461); - EXPECT_EQ(ext_render_message.crop_from_location().height(), 720); - EXPECT_EQ(ext_render_message.render_to_location().x(), 0); - EXPECT_EQ(ext_render_message.render_to_location().y(), 0); - EXPECT_EQ(ext_render_message.render_to_location().width(), 720); - EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); - } -} - -// Checks external render message with default poly path solver without video -// input. -TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPathNoVideo) { - const CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kExternalRenderConfigNoVideo, kTargetWidth, - kTargetHeight, kKeyFrameWidth, kKeyFrameHeight)); - auto runner = absl::make_unique(config); - const int num_frames = kSceneSize; - AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, 1, runner->MutableInputs()); - - MP_EXPECT_OK(runner->Run()); - const auto& outputs = runner->Outputs(); - const auto& ext_render_per_frame = - outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets; - EXPECT_EQ(ext_render_per_frame.size(), num_frames); - - for (int i = 0; i < num_frames - 1; ++i) { - const auto& ext_render_message = - ext_render_per_frame[i].Get(); - EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000); - EXPECT_EQ(ext_render_message.crop_from_location().x(), 725); - EXPECT_EQ(ext_render_message.crop_from_location().y(), 0); - EXPECT_EQ(ext_render_message.crop_from_location().width(), 461); - EXPECT_EQ(ext_render_message.crop_from_location().height(), 720); - EXPECT_EQ(ext_render_message.render_to_location().x(), 0); - EXPECT_EQ(ext_render_message.render_to_location().y(), 0); - EXPECT_EQ(ext_render_message.render_to_location().width(), 720); - EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); - } -} - -// Checks external render message with kinematic path solver without video -// input. -TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPathNoVideo) { - CalculatorGraphConfig::Node config = - ParseTextProtoOrDie( - absl::Substitute(kExternalRenderConfigNoVideo, kTargetWidth, - kTargetHeight, kKeyFrameWidth, kKeyFrameHeight)); - auto* options = config.mutable_options()->MutableExtension( - SceneCroppingCalculatorOptions::ext); - auto* kinematic_options = - options->mutable_camera_motion_options()->mutable_kinematic_options(); - kinematic_options->set_min_motion_to_reframe(1.2); - kinematic_options->set_max_velocity(2.0); - - auto runner = absl::make_unique(config); - const int num_frames = kSceneSize; - AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, - kKeyFrameHeight, 1, runner->MutableInputs()); - - MP_EXPECT_OK(runner->Run()); - const auto& outputs = runner->Outputs(); - const auto& ext_render_per_frame = - outputs.Tag("EXTERNAL_RENDERING_PER_FRAME").packets; - EXPECT_EQ(ext_render_per_frame.size(), num_frames); - - for (int i = 0; i < num_frames - 1; ++i) { - const auto& ext_render_message = - ext_render_per_frame[i].Get(); - EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000); - EXPECT_EQ(ext_render_message.crop_from_location().x(), 725); - EXPECT_EQ(ext_render_message.crop_from_location().y(), 0); - EXPECT_EQ(ext_render_message.crop_from_location().width(), 461); - EXPECT_EQ(ext_render_message.crop_from_location().height(), 720); - EXPECT_EQ(ext_render_message.render_to_location().x(), 0); - EXPECT_EQ(ext_render_message.render_to_location().y(), 0); - EXPECT_EQ(ext_render_message.render_to_location().width(), 720); - EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); - } -} -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc deleted file mode 100644 index 299f60b10..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -using mediapipe::ImageFrame; -using mediapipe::PacketTypeSet; - -// IO labels. -constexpr char kVideoInputTag[] = "VIDEO"; -constexpr char kShotChangeTag[] = "IS_SHOT_CHANGE"; -// Histogram settings. -const int kSaturationBins = 8; -const int kHistogramChannels[] = {0, 1, 2}; -const int kHistogramBinNum[] = {kSaturationBins, kSaturationBins, - kSaturationBins}; -const float kRange[] = {0, 256}; -const float* kHistogramRange[] = {kRange, kRange, kRange}; - -namespace mediapipe { -namespace autoflip { - -// This calculator computes a shot (or scene) change within a video. It works -// by computing a 3d color histogram and comparing this frame-to-frame. Settings -// to control the shot change logic are presented in the options proto. -// -// Example: -// node { -// calculator: "ShotBoundaryCalculator" -// input_stream: "VIDEO:camera_frames" -// output_stream: "IS_SHOT_CHANGE:is_shot" -// } -class ShotBoundaryCalculator : public mediapipe::CalculatorBase { - public: - ShotBoundaryCalculator() {} - ShotBoundaryCalculator(const ShotBoundaryCalculator&) = delete; - ShotBoundaryCalculator& operator=(const ShotBoundaryCalculator&) = delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - - private: - // Computes the histogram of an image. - void ComputeHistogram(const cv::Mat& image, cv::Mat* image_histogram); - // Transmits signal to next calculator. - void Transmit(mediapipe::CalculatorContext* cc, bool is_shot_change); - // Calculator options. - ShotBoundaryCalculatorOptions options_; - // Last time a shot was detected. - Timestamp last_shot_timestamp_; - // Defines if the calculator has received a frame yet. - bool init_; - // Histogram from the last frame. - cv::Mat last_histogram_; - // History of histogram motion. - std::deque motion_history_; -}; -REGISTER_CALCULATOR(ShotBoundaryCalculator); - -void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image, - cv::Mat* image_histogram) { - cv::Mat equalized_image; - cv::cvtColor(image.clone(), equalized_image, cv::COLOR_RGB2GRAY); - - double min, max; - cv::minMaxLoc(equalized_image, &min, &max); - - if (options_.equalize_histogram()) { - cv::equalizeHist(equalized_image, equalized_image); - } - - cv::calcHist(&image, 1, kHistogramChannels, cv::Mat(), *image_histogram, 2, - kHistogramBinNum, kHistogramRange, true, false); -} - -absl::Status ShotBoundaryCalculator::Open(mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - last_shot_timestamp_ = Timestamp(0); - init_ = false; - return absl::OkStatus(); -} - -void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc, - bool is_shot_change) { - if ((cc->InputTimestamp() - last_shot_timestamp_).Seconds() < - options_.min_shot_span()) { - is_shot_change = false; - } - if (is_shot_change) { - LOG(INFO) << "Shot change at: " << cc->InputTimestamp().Seconds() - << " seconds."; - cc->Outputs() - .Tag(kShotChangeTag) - .AddPacket(Adopt(std::make_unique(true).release()) - .At(cc->InputTimestamp())); - } else if (!options_.output_only_on_change()) { - cc->Outputs() - .Tag(kShotChangeTag) - .AddPacket(Adopt(std::make_unique(false).release()) - .At(cc->InputTimestamp())); - } -} - -absl::Status ShotBoundaryCalculator::Process(mediapipe::CalculatorContext* cc) { - // Connect to input frame and make a mutable copy. - cv::Mat frame_org = mediapipe::formats::MatView( - &cc->Inputs().Tag(kVideoInputTag).Get()); - cv::Mat frame = frame_org.clone(); - - // Extract histogram from the current frame. - cv::Mat current_histogram; - ComputeHistogram(frame, ¤t_histogram); - - if (!init_) { - last_histogram_ = current_histogram; - init_ = true; - Transmit(cc, false); - return absl::OkStatus(); - } - - double current_motion_estimate = - 1 - cv::compareHist(current_histogram, last_histogram_, CV_COMP_CORREL); - last_histogram_ = current_histogram; - motion_history_.push_front(current_motion_estimate); - - if (motion_history_.size() != options_.window_size()) { - Transmit(cc, false); - return absl::OkStatus(); - } - - // Shot detection algorithm is a mixture of adaptive (controlled with - // shot_measure) and hard thresholds. In saturation it uses hard thresholds - // to account for black startups, shot cuts across high motion etc. - // In the operating region it uses an adaptive threshold to tune motion vs. - // cut boundary. - double current_max = - *std::max_element(motion_history_.begin(), motion_history_.end()); - double shot_measure = current_motion_estimate / current_max; - - if ((shot_measure > options_.min_shot_measure() && - current_motion_estimate > options_.min_motion_with_shot_measure()) || - current_motion_estimate > options_.min_motion()) { - Transmit(cc, true); - last_shot_timestamp_ = cc->InputTimestamp(); - } else { - Transmit(cc, false); - } - - // Store histogram for next frame. - last_histogram_ = current_histogram; - motion_history_.pop_back(); - return absl::OkStatus(); -} - -absl::Status ShotBoundaryCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - cc->Inputs().Tag(kVideoInputTag).Set(); - cc->Outputs().Tag(kShotChangeTag).Set(); - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto deleted file mode 100644 index b0f3bd9d2..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -message ShotBoundaryCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional ShotBoundaryCalculatorOptions ext = 281194049; - } - // Parameters to shot detection algorithm. All the constraints (the fields - // named with 'min_') need to be satisfied for a frame to be a shot boundary. - // - // Minimum motion to be considered as a shot boundary frame. - optional double min_motion = 1 [default = 0.2]; - // Minimum number of shot duration (in seconds). - optional double min_shot_span = 2 [default = 2]; - // A window for computing shot measure (see the definition in min_shot_measure - // field). - optional int32 window_size = 3 [default = 7]; - // Minimum shot measure to be considered as a shot boundary frame. - // Must also satisfy the min_motion_with_shot_measure constraint. - // The shot measure is defined as the ratio of the motion of the - // current frame to the maximum motion of the frames in the window (defined - // as window_size). - optional double min_shot_measure = 4 [default = 10]; - // Minimum motion to be considered as a shot boundary frame. - // Must also satisfy the min_shot_measure constraint. - optional double min_motion_with_shot_measure = 5 [default = 0.05]; - // Only send results if the shot value is true. - optional bool output_only_on_change = 6 [default = true]; - // Perform histogram equalization before computing keypoints/features. - optional bool equalize_histogram = 7 [default = false]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc deleted file mode 100644 index a26c7e44c..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator_test.cc +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/flags/flag.h" -#include "absl/strings/string_view.h" -#include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -using mediapipe::Adopt; -using mediapipe::CalculatorGraphConfig; -using mediapipe::CalculatorRunner; -using mediapipe::ImageFormat; -using mediapipe::ImageFrame; -using mediapipe::PacketTypeSet; -using mediapipe::ParseTextProtoOrDie; -using mediapipe::Timestamp; - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfig[] = R"( - calculator: "ShotBoundaryCalculator" - input_stream: "VIDEO:camera_frames" - output_stream: "IS_SHOT_CHANGE:is_shot" - )"; -const int kTestFrameWidth = 640; -const int kTestFrameHeight = 480; - -void AddFrames(const int number_of_frames, const std::set& skip_frames, - CalculatorRunner* runner) { - cv::Mat image = - cv::imread(file::JoinPath("./", - "/mediapipe/examples/desktop/" - "autoflip/calculators/testdata/dino.jpg")); - - for (int i = 0; i < number_of_frames; i++) { - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kTestFrameWidth, kTestFrameHeight); - cv::Mat input_mat = mediapipe::formats::MatView(input_frame.get()); - input_mat.setTo(cv::Scalar(0, 0, 0)); - cv::Mat sub_image = - image(cv::Rect(i, i, kTestFrameWidth, kTestFrameHeight)); - cv::Mat frame_area = - input_mat(cv::Rect(0, 0, sub_image.cols, sub_image.rows)); - if (skip_frames.count(i) < 1) { - sub_image.copyTo(frame_area); - } - runner->MutableInputs()->Tag("VIDEO").packets.push_back( - Adopt(input_frame.release()).At(Timestamp(i * 1000000))); - } -} - -void CheckOutput(const int number_of_frames, const std::set& shot_frames, - const std::vector& output_packets) { - ASSERT_EQ(number_of_frames, output_packets.size()); - for (int i = 0; i < number_of_frames; i++) { - if (shot_frames.count(i) < 1) { - EXPECT_FALSE(output_packets[i].Get()); - } else { - EXPECT_TRUE(output_packets[i].Get()); - } - } -} - -TEST(ShotBoundaryCalculatorTest, NoShotChange) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfig); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_output_only_on_change(false); - auto runner = ::absl::make_unique(node); - - AddFrames(10, {}, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckOutput(10, {}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets); -} - -TEST(ShotBoundaryCalculatorTest, ShotChangeSingle) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfig); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_output_only_on_change(false); - auto runner = ::absl::make_unique(node); - - AddFrames(20, {10}, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckOutput(20, {10}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets); -} - -TEST(ShotBoundaryCalculatorTest, ShotChangeDouble) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfig); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_output_only_on_change(false); - auto runner = ::absl::make_unique(node); - - AddFrames(20, {14, 17}, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckOutput(20, {14, 17}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets); -} - -TEST(ShotBoundaryCalculatorTest, ShotChangeFiltered) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfig); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_min_shot_span(5); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_output_only_on_change(false); - - auto runner = ::absl::make_unique(node); - - AddFrames(24, {16, 19}, runner.get()); - MP_ASSERT_OK(runner->Run()); - CheckOutput(24, {16}, runner->Outputs().Tag("IS_SHOT_CHANGE").packets); -} - -TEST(ShotBoundaryCalculatorTest, ShotChangeSingleOnOnChange) { - CalculatorGraphConfig::Node node = - ParseTextProtoOrDie(kConfig); - node.mutable_options() - ->MutableExtension(ShotBoundaryCalculatorOptions::ext) - ->set_output_only_on_change(true); - auto runner = ::absl::make_unique(node); - - AddFrames(20, {15}, runner.get()); - MP_ASSERT_OK(runner->Run()); - auto output_packets = runner->Outputs().Tag("IS_SHOT_CHANGE").packets; - ASSERT_EQ(output_packets.size(), 1); - ASSERT_EQ(output_packets[0].Get(), true); - ASSERT_EQ(output_packets[0].Timestamp().Value(), 15000000); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc deleted file mode 100644 index 37643b5d1..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.cc +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -using mediapipe::Packet; -using mediapipe::PacketTypeSet; -using mediapipe::autoflip::DetectionSet; -using mediapipe::autoflip::SalientRegion; -using mediapipe::autoflip::SignalType; - -constexpr char kIsShotBoundaryTag[] = "IS_SHOT_BOUNDARY"; -constexpr char kSignalInputsTag[] = "SIGNAL"; -constexpr char kOutputTag[] = "OUTPUT"; - -namespace mediapipe { -namespace autoflip { - -struct InputSignal { - SalientRegion signal; - int source; -}; - -struct Frame { - std::vector input_detections; - mediapipe::Timestamp time; -}; - -// This calculator takes one scene change signal (optional, see below) and an -// arbitrary number of detection signals and outputs a single list of -// detections. The scores for the detections can be re-normalized using the -// options proto. Additionally, if a detection has a consistent tracking id -// during a scene the score for that detection is averaged over the whole scene. -// -// Example (ordered interface): -// node { -// calculator: "SignalFusingCalculator" -// input_stream: "scene_change" (required for ordered interface) -// input_stream: "detection_faces" -// input_stream: "detection_custom_text" -// output_stream: "salient_region" -// options:{ -// [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{ -// signal_settings{ -// type: {standard: FACE} -// min_score: 0.5 -// max_score: 0.6 -// } -// signal_settings{ -// type: {custom: "custom_text"} -// min_score: 0.9 -// max_score: 1.0 -// } -// } -// } -// } -// -// Example (tag interface): -// node { -// calculator: "SignalFusingCalculator" -// input_stream: "IS_SHOT_BOUNDARY:scene_change" (optional) -// input_stream: "SIGNAL:0:detection_faces" -// input_stream: "SIGNAL:1:detection_custom_text" -// output_stream: "OUTPUT:salient_region" -// options:{ -// [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{ -// signal_settings{ -// type: {standard: FACE} -// min_score: 0.5 -// max_score: 0.6 -// } -// signal_settings{ -// type: {custom: "custom_text"} -// min_score: 0.9 -// max_score: 1.0 -// } -// } -// } -// } -class SignalFusingCalculator : public mediapipe::CalculatorBase { - public: - SignalFusingCalculator() - : tag_input_interface_(false), process_by_scene_(true) {} - SignalFusingCalculator(const SignalFusingCalculator&) = delete; - SignalFusingCalculator& operator=(const SignalFusingCalculator&) = delete; - - static absl::Status GetContract(mediapipe::CalculatorContract* cc); - absl::Status Open(mediapipe::CalculatorContext* cc) override; - absl::Status Process(mediapipe::CalculatorContext* cc) override; - absl::Status Close(mediapipe::CalculatorContext* cc) override; - - private: - absl::Status ProcessScene(mediapipe::CalculatorContext* cc); - std::vector GetSignalPackets(mediapipe::CalculatorContext* cc); - SignalFusingCalculatorOptions options_; - std::map settings_by_type_; - std::vector scene_frames_; - bool tag_input_interface_; - bool process_by_scene_; -}; -REGISTER_CALCULATOR(SignalFusingCalculator); - -namespace { -std::string CreateSettingsKey(const SignalType& signal_type) { - if (signal_type.has_standard()) { - return "standard_" + std::to_string(signal_type.standard()); - } else { - return "custom_" + signal_type.custom(); - } -} -std::string CreateKey(const InputSignal& detection) { - std::string id_source = std::to_string(detection.source); - std::string id_signal = std::to_string(detection.signal.tracking_id()); - std::string id = id_source + ":" + id_signal; - return id; -} -void SetupTagInput(mediapipe::CalculatorContract* cc) { - if (cc->Inputs().HasTag(kIsShotBoundaryTag)) { - cc->Inputs().Tag(kIsShotBoundaryTag).Set(); - } - for (int i = 0; i < cc->Inputs().NumEntries(kSignalInputsTag); i++) { - cc->Inputs().Get(kSignalInputsTag, i).Set(); - } - cc->Outputs().Tag(kOutputTag).Set(); -} - -void SetupOrderedInput(mediapipe::CalculatorContract* cc) { - cc->Inputs().Index(0).Set(); - for (int i = 1; i < cc->Inputs().NumEntries(); ++i) { - cc->Inputs().Index(i).Set(); - } - cc->Outputs().Index(0).Set(); -} -} // namespace - -absl::Status SignalFusingCalculator::Open(mediapipe::CalculatorContext* cc) { - options_ = cc->Options(); - for (const auto& setting : options_.signal_settings()) { - settings_by_type_[CreateSettingsKey(setting.type())] = setting; - } - if (cc->Inputs().HasTag(kSignalInputsTag)) { - tag_input_interface_ = true; - if (!cc->Inputs().HasTag(kIsShotBoundaryTag)) { - process_by_scene_ = false; - } - } - return absl::OkStatus(); -} - -absl::Status SignalFusingCalculator::Close(mediapipe::CalculatorContext* cc) { - if (!scene_frames_.empty()) { - MP_RETURN_IF_ERROR(ProcessScene(cc)); - scene_frames_.clear(); - } - return absl::OkStatus(); -} - -absl::Status SignalFusingCalculator::ProcessScene( - mediapipe::CalculatorContext* cc) { - std::map detection_count; - std::map multiframe_score; - // Create a unified score for all items with temporal ids. - for (const Frame& frame : scene_frames_) { - for (const auto& detection : frame.input_detections) { - if (detection.signal.has_tracking_id()) { - // Create key for each detector type - if (detection_count.find(CreateKey(detection)) == - detection_count.end()) { - multiframe_score[CreateKey(detection)] = 0.0; - detection_count[CreateKey(detection)] = 0; - } - multiframe_score[CreateKey(detection)] += detection.signal.score(); - detection_count[CreateKey(detection)]++; - } - } - } - // Average scores. - for (auto iterator = multiframe_score.begin(); - iterator != multiframe_score.end(); iterator++) { - multiframe_score[iterator->first] = - iterator->second / detection_count[iterator->first]; - } - // Process detections. - for (const Frame& frame : scene_frames_) { - std::unique_ptr processed_detections(new DetectionSet()); - for (auto detection : frame.input_detections) { - float score = detection.signal.score(); - if (detection.signal.has_tracking_id()) { - std::string id_source = std::to_string(detection.source); - std::string id_signal = std::to_string(detection.signal.tracking_id()); - std::string id = id_source + ":" + id_signal; - score = multiframe_score[id]; - } - // Normalize within range. - float min_value = 0.0; - float max_value = 1.0; - - auto settings_it = settings_by_type_.find( - CreateSettingsKey(detection.signal.signal_type())); - if (settings_it != settings_by_type_.end()) { - min_value = settings_it->second.min_score(); - max_value = settings_it->second.max_score(); - detection.signal.set_is_required(settings_it->second.is_required()); - detection.signal.set_only_required(settings_it->second.only_required()); - } - - float final_score = score * (max_value - min_value) + min_value; - detection.signal.set_score(final_score); - *processed_detections->add_detections() = detection.signal; - } - if (tag_input_interface_) { - cc->Outputs() - .Tag(kOutputTag) - .Add(processed_detections.release(), frame.time); - } else { - cc->Outputs().Index(0).Add(processed_detections.release(), frame.time); - } - } - - return absl::OkStatus(); -} - -std::vector SignalFusingCalculator::GetSignalPackets( - mediapipe::CalculatorContext* cc) { - std::vector signal_packets; - if (tag_input_interface_) { - for (int i = 0; i < cc->Inputs().NumEntries(kSignalInputsTag); i++) { - const Packet& packet = cc->Inputs().Get(kSignalInputsTag, i).Value(); - signal_packets.push_back(packet); - } - } else { - for (int i = 1; i < cc->Inputs().NumEntries(); i++) { - const Packet& packet = cc->Inputs().Index(i).Value(); - signal_packets.push_back(packet); - } - } - return signal_packets; -} - -absl::Status SignalFusingCalculator::Process(mediapipe::CalculatorContext* cc) { - bool is_boundary = false; - if (process_by_scene_) { - const auto& shot_tag = (tag_input_interface_) - ? cc->Inputs().Tag(kIsShotBoundaryTag) - : cc->Inputs().Index(0); - if (!shot_tag.Value().IsEmpty()) { - is_boundary = shot_tag.Get(); - } - } - - if (is_boundary) { - MP_RETURN_IF_ERROR(ProcessScene(cc)); - scene_frames_.clear(); - } - - Frame frame; - const auto& signal_packets = GetSignalPackets(cc); - for (int i = 0; i < signal_packets.size(); i++) { - const Packet& packet = signal_packets[i]; - if (packet.IsEmpty()) { - continue; - } - const auto& detection_set = packet.Get(); - for (const auto& detection : detection_set.detections()) { - InputSignal input; - input.signal = detection; - input.source = i; - frame.input_detections.push_back(input); - } - } - frame.time = cc->InputTimestamp(); - scene_frames_.push_back(frame); - - // Flush buffer on same input if it exceeds max_scene_size or if there is not - // shot input information. - if (scene_frames_.size() > options_.max_scene_size() || !process_by_scene_) { - MP_RETURN_IF_ERROR(ProcessScene(cc)); - scene_frames_.clear(); - } - - return absl::OkStatus(); -} - -absl::Status SignalFusingCalculator::GetContract( - mediapipe::CalculatorContract* cc) { - if (cc->Inputs().NumEntries(kSignalInputsTag) > 0) { - SetupTagInput(cc); - } else { - SetupOrderedInput(cc); - } - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto deleted file mode 100644 index 3f14935ba..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.proto +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto"; -import "mediapipe/framework/calculator.proto"; - -// Next tag: 3 -message SignalFusingCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional SignalFusingCalculatorOptions ext = 280092372; - } - // Setting related to each type of signal this calculator could process. - repeated SignalSettings signal_settings = 1; - - // Force a flush of the frame buffer after this number of frames. - optional int32 max_scene_size = 2 [default = 600]; -} - -// Next tag: 6 -message SignalSettings { - // The type of signal these settings pertain to. - optional SignalType type = 1; - - // Force a normalized incoming score to be re-normalized to within this range. - // (set values to min:0 and max:1 for no change in the incoming score) - // Values must be between 0-1, min must be less than max. - // - // Example of score adjustment: - // Incoming OCR score: .7 - // Min OCR Score: .9 - // Max OCR Score: 1.0 - // --Result: .97 - optional float min_score = 2 [default = 0]; - optional float max_score = 3 [default = 1.0]; - - // Is this signal required within the output cropped video? If it is it will - // be included or the video will be marked as failed to convert. - optional bool is_required = 4 [default = false]; - - // When used with ContentZoomingCalculator, this flag can be set indicating - // that areas outside of these salient regions can be cropped from the frame. - // When no salient regions have this flag set true, no zooming is performed. - // When one or more salient regions have this flag set true, the max zoom - // value will be used that keeps all “only_required” detections within view. - // The ContentZoomingCalculator currently supports zooming by finding the size - // of non-salient top/bottom borders regions and provides this information to - // the SceneCroppingCalculator for reframing. - optional bool only_required = 5 [default = false]; -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc deleted file mode 100644 index 79fab5a0f..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator_test.cc +++ /dev/null @@ -1,566 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/strings/string_view.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/calculators/signal_fusing_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -using mediapipe::autoflip::DetectionSet; - -namespace mediapipe { -namespace autoflip { -namespace { - -const char kConfigA[] = R"( - calculator: "SignalFusingCalculator" - input_stream: "scene_change" - input_stream: "detection_set_a" - input_stream: "detection_set_b" - output_stream: "salient_region" - options:{ - [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{ - signal_settings{ - type: {standard: FACE_FULL} - min_score: 0.5 - max_score: 0.6 - } - signal_settings{ - type: {standard: TEXT} - min_score: 0.9 - max_score: 1.0 - } - } - })"; - -const char kConfigB[] = R"( - calculator: "SignalFusingCalculator" - input_stream: "scene_change" - input_stream: "detection_set_a" - input_stream: "detection_set_b" - input_stream: "detection_set_c" - output_stream: "salient_region" - options:{ - [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{ - signal_settings{ - type: {standard: FACE_FULL} - min_score: 0.5 - max_score: 0.6 - } - signal_settings{ - type: {custom: "text"} - min_score: 0.9 - max_score: 1.0 - } - signal_settings{ - type: {standard: LOGO} - min_score: 0.1 - max_score: 0.3 - } - } - })"; - -const char kConfigC[] = R"( - calculator: "SignalFusingCalculator" - input_stream: "IS_SHOT_BOUNDARY:scene_change" - input_stream: "SIGNAL:0:detection_set_a" - input_stream: "SIGNAL:1:detection_set_b" - output_stream: "OUTPUT:salient_region" - options:{ - [mediapipe.autoflip.SignalFusingCalculatorOptions.ext]:{ - signal_settings{ - type: {standard: FACE_FULL} - min_score: 0.5 - max_score: 0.6 - } - signal_settings{ - type: {standard: TEXT} - min_score: 0.9 - max_score: 1.0 - } - } - })"; - -TEST(SignalFusingCalculatorTest, TwoInputNoTracking) { - auto runner = absl::make_unique( - ParseTextProtoOrDie(kConfigA)); - - auto input_border = absl::make_unique(false); - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(input_border.release()).At(Timestamp(0))); - - auto input_face = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.5 - signal_type: { standard: FACE_FULL } - } - detections { - score: 0.3 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs()->Index(1).packets.push_back( - Adopt(input_face.release()).At(Timestamp(0))); - - auto input_ocr = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { standard: TEXT } - } - detections { - score: 0.9 - signal_type: { standard: TEXT } - } - )pb")); - - runner->MutableInputs()->Index(2).packets.push_back( - Adopt(input_ocr.release()).At(Timestamp(0))); - - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Index(0).packets; - const auto& detection_set = output_packets[0].Get(); - - ASSERT_EQ(detection_set.detections().size(), 4); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99); -} - -TEST(SignalFusingCalculatorTest, TwoInputShotLabeledTags) { - auto runner = absl::make_unique( - ParseTextProtoOrDie(kConfigC)); - - auto input_shot = absl::make_unique(false); - runner->MutableInputs() - ->Tag("IS_SHOT_BOUNDARY") - .packets.push_back(Adopt(input_shot.release()).At(Timestamp(0))); - - auto input_face = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.5 - signal_type: { standard: FACE_FULL } - } - detections { - score: 0.3 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs() - ->Get("SIGNAL", 0) - .packets.push_back(Adopt(input_face.release()).At(Timestamp(0))); - - auto input_ocr = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { standard: TEXT } - } - detections { - score: 0.9 - signal_type: { standard: TEXT } - } - )pb")); - - runner->MutableInputs() - ->Get("SIGNAL", 1) - .packets.push_back(Adopt(input_ocr.release()).At(Timestamp(0))); - - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("OUTPUT").packets; - const auto& detection_set = output_packets[0].Get(); - - ASSERT_EQ(detection_set.detections().size(), 4); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99); -} - -TEST(SignalFusingCalculatorTest, TwoInputNoShotLabeledTags) { - auto runner = absl::make_unique( - ParseTextProtoOrDie(kConfigC)); - - auto input_face = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.5 - signal_type: { standard: FACE_FULL } - } - detections { - score: 0.3 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs() - ->Get("SIGNAL", 0) - .packets.push_back(Adopt(input_face.release()).At(Timestamp(0))); - - auto input_ocr = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { standard: TEXT } - } - detections { - score: 0.9 - signal_type: { standard: TEXT } - } - )pb")); - - runner->MutableInputs() - ->Get("SIGNAL", 1) - .packets.push_back(Adopt(input_ocr.release()).At(Timestamp(0))); - - MP_ASSERT_OK(runner->Run()); - - const std::vector& output_packets = - runner->Outputs().Tag("OUTPUT").packets; - const auto& detection_set = output_packets[0].Get(); - - ASSERT_EQ(detection_set.detections().size(), 4); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), .55); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), .53); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), .93); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), .99); -} - -TEST(SignalFusingCalculatorTest, ThreeInputTracking) { - auto runner = absl::make_unique( - ParseTextProtoOrDie(kConfigB)); - - auto input_border_0 = absl::make_unique(false); - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(input_border_0.release()).At(Timestamp(0))); - - // Time zero. - auto input_face_0 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.2 - signal_type: { standard: FACE_FULL } - tracking_id: 0 - } - detections { - score: 0.0 - signal_type: { standard: FACE_FULL } - tracking_id: 1 - } - detections { - score: 0.1 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs()->Index(1).packets.push_back( - Adopt(input_face_0.release()).At(Timestamp(0))); - - auto input_ocr_0 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.2 - signal_type: { custom: "text" } - } - )pb")); - - runner->MutableInputs()->Index(2).packets.push_back( - Adopt(input_ocr_0.release()).At(Timestamp(0))); - - auto input_agn_0 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { standard: LOGO } - tracking_id: 0 - } - )pb")); - - runner->MutableInputs()->Index(3).packets.push_back( - Adopt(input_agn_0.release()).At(Timestamp(0))); - - // Time one - auto input_border_1 = absl::make_unique(false); - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(input_border_1.release()).At(Timestamp(1))); - - auto input_face_1 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.7 - signal_type: { standard: FACE_FULL } - tracking_id: 0 - } - detections { - score: 0.9 - signal_type: { standard: FACE_FULL } - tracking_id: 1 - } - detections { - score: 0.2 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs()->Index(1).packets.push_back( - Adopt(input_face_1.release()).At(Timestamp(1))); - - auto input_ocr_1 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { custom: "text" } - } - )pb")); - - runner->MutableInputs()->Index(2).packets.push_back( - Adopt(input_ocr_1.release()).At(Timestamp(1))); - - auto input_agn_1 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { standard: LOGO } - tracking_id: 0 - } - )pb")); - - runner->MutableInputs()->Index(3).packets.push_back( - Adopt(input_agn_1.release()).At(Timestamp(1))); - - // Time two - auto input_border_2 = absl::make_unique(false); - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(input_border_2.release()).At(Timestamp(2))); - - auto input_face_2 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.8 - signal_type: { standard: FACE_FULL } - tracking_id: 0 - } - detections { - score: 0.9 - signal_type: { standard: FACE_FULL } - tracking_id: 1 - } - detections { - score: 0.3 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs()->Index(1).packets.push_back( - Adopt(input_face_2.release()).At(Timestamp(2))); - - auto input_ocr_2 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.3 - signal_type: { custom: "text" } - } - )pb")); - - runner->MutableInputs()->Index(2).packets.push_back( - Adopt(input_ocr_2.release()).At(Timestamp(2))); - - auto input_agn_2 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.9 - signal_type: { standard: LOGO } - tracking_id: 0 - } - )pb")); - - runner->MutableInputs()->Index(3).packets.push_back( - Adopt(input_agn_2.release()).At(Timestamp(2))); - - // Time three (new scene) - auto input_border_3 = absl::make_unique(true); - runner->MutableInputs()->Index(0).packets.push_back( - Adopt(input_border_3.release()).At(Timestamp(3))); - - auto input_face_3 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.2 - signal_type: { standard: FACE_FULL } - tracking_id: 0 - } - detections { - score: 0.3 - signal_type: { standard: FACE_FULL } - tracking_id: 1 - } - detections { - score: 0.4 - signal_type: { standard: FACE_FULL } - } - )pb")); - - runner->MutableInputs()->Index(1).packets.push_back( - Adopt(input_face_3.release()).At(Timestamp(3))); - - auto input_ocr_3 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.5 - signal_type: { custom: "text" } - } - )pb")); - - runner->MutableInputs()->Index(2).packets.push_back( - Adopt(input_ocr_3.release()).At(Timestamp(3))); - - auto input_agn_3 = - absl::make_unique(ParseTextProtoOrDie( - R"pb( - detections { - score: 0.6 - signal_type: { standard: LOGO } - tracking_id: 0 - } - )pb")); - - runner->MutableInputs()->Index(3).packets.push_back( - Adopt(input_agn_3.release()).At(Timestamp(3))); - - MP_ASSERT_OK(runner->Run()); - - // Check time 0 - std::vector output_packets = runner->Outputs().Index(0).packets; - DetectionSet detection_set = output_packets[0].Get(); - - float face_id_0 = (.2 + .7 + .8) / 3; - face_id_0 = face_id_0 * .1 + .5; - float face_id_1 = (0.0 + .9 + .9) / 3; - face_id_1 = face_id_1 * .1 + .5; - float face_3 = 0.1; - face_3 = face_3 * .1 + .5; - float ocr_1 = 0.2; - ocr_1 = ocr_1 * .1 + .9; - float agn_1 = (.3 + .3 + .9) / 3; - agn_1 = agn_1 * .2 + .1; - - ASSERT_EQ(detection_set.detections().size(), 5); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1); - EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1); - - // Check time 1 - detection_set = output_packets[1].Get(); - - face_id_0 = (.2 + .7 + .8) / 3; - face_id_0 = face_id_0 * .1 + .5; - face_id_1 = (0.0 + .9 + .9) / 3; - face_id_1 = face_id_1 * .1 + .5; - face_3 = 0.2; - face_3 = face_3 * .1 + .5; - ocr_1 = 0.3; - ocr_1 = ocr_1 * .1 + .9; - agn_1 = (.3 + .3 + .9) / 3; - agn_1 = agn_1 * .2 + .1; - - ASSERT_EQ(detection_set.detections().size(), 5); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1); - EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1); - - // Check time 2 - detection_set = output_packets[2].Get(); - - face_id_0 = (.2 + .7 + .8) / 3; - face_id_0 = face_id_0 * .1 + .5; - face_id_1 = (0.0 + .9 + .9) / 3; - face_id_1 = face_id_1 * .1 + .5; - face_3 = 0.3; - face_3 = face_3 * .1 + .5; - ocr_1 = 0.3; - ocr_1 = ocr_1 * .1 + .9; - agn_1 = (.3 + .3 + .9) / 3; - agn_1 = agn_1 * .2 + .1; - - ASSERT_EQ(detection_set.detections().size(), 5); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1); - EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1); - - // Check time 3 (new scene) - detection_set = output_packets[3].Get(); - - face_id_0 = 0.2; - face_id_0 = face_id_0 * .1 + .5; - face_id_1 = 0.3; - face_id_1 = face_id_1 * .1 + .5; - face_3 = 0.4; - face_3 = face_3 * .1 + .5; - ocr_1 = 0.5; - ocr_1 = ocr_1 * .1 + .9; - agn_1 = .6; - agn_1 = agn_1 * .2 + .1; - - ASSERT_EQ(detection_set.detections().size(), 5); - EXPECT_FLOAT_EQ(detection_set.detections(0).score(), face_id_0); - EXPECT_FLOAT_EQ(detection_set.detections(1).score(), face_id_1); - EXPECT_FLOAT_EQ(detection_set.detections(2).score(), face_3); - EXPECT_FLOAT_EQ(detection_set.detections(3).score(), ocr_1); - EXPECT_FLOAT_EQ(detection_set.detections(4).score(), agn_1); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD b/mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD deleted file mode 100644 index cd99c8244..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/testdata/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -filegroup( - name = "test_images", - srcs = [ - "dino.jpg", - ], - visibility = ["//visibility:public"], -) diff --git a/mediapipe/examples/desktop/autoflip/calculators/testdata/dino.jpg b/mediapipe/examples/desktop/autoflip/calculators/testdata/dino.jpg deleted file mode 100644 index 0b00f50f1..000000000 Binary files a/mediapipe/examples/desktop/autoflip/calculators/testdata/dino.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc deleted file mode 100644 index 8d67eb8f0..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/status_builder.h" - -namespace mediapipe { -namespace autoflip { -namespace { -constexpr char kInputFrameTag[] = "INPUT_FRAMES"; -constexpr char kOutputFrameTag[] = "OUTPUT_FRAMES"; -} // namespace - -// This calculator filters out frames based on criteria specified in the -// options. One use case is to filter based on the aspect ratio. Future work -// can implement more filter types. -// -// Input: Video frames. -// Output: Video frames that pass all filters. -// -// Example config: -// node { -// calculator: "VideoFilteringCalculator" -// input_stream: "INPUT_FRAMES:frames" -// output_stream: "OUTPUT_FRAMES:output_frames" -// options: { -// [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: { -// fail_if_any: true -// aspect_ratio_filter { -// target_width: 400 -// target_height: 600 -// filter_type: UPPER_ASPECT_RATIO_THRESHOLD -// } -// } -// } -// } -class VideoFilteringCalculator : public CalculatorBase { - public: - VideoFilteringCalculator() = default; - ~VideoFilteringCalculator() override = default; - - static absl::Status GetContract(CalculatorContract* cc); - - absl::Status Process(CalculatorContext* cc) override; -}; -REGISTER_CALCULATOR(VideoFilteringCalculator); - -absl::Status VideoFilteringCalculator::GetContract(CalculatorContract* cc) { - cc->Inputs().Tag(kInputFrameTag).Set(); - cc->Outputs().Tag(kOutputFrameTag).Set(); - return absl::OkStatus(); -} - -absl::Status VideoFilteringCalculator::Process(CalculatorContext* cc) { - const auto& options = cc->Options(); - - const Packet& input_packet = cc->Inputs().Tag(kInputFrameTag).Value(); - const ImageFrame& frame = input_packet.Get(); - - RET_CHECK(options.has_aspect_ratio_filter()); - const auto filter_type = options.aspect_ratio_filter().filter_type(); - RET_CHECK_NE( - filter_type, - VideoFilteringCalculatorOptions::AspectRatioFilter::UNKNOWN_FILTER_TYPE); - if (filter_type == - VideoFilteringCalculatorOptions::AspectRatioFilter::NO_FILTERING) { - cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet); - return absl::OkStatus(); - } - const int target_width = options.aspect_ratio_filter().target_width(); - const int target_height = options.aspect_ratio_filter().target_height(); - RET_CHECK_GT(target_width, 0); - RET_CHECK_GT(target_height, 0); - - bool should_pass = false; - cv::Mat frame_mat = mediapipe::formats::MatView(&frame); - const double ratio = static_cast(frame_mat.cols) / frame_mat.rows; - const double target_ratio = static_cast(target_width) / target_height; - if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter:: - UPPER_ASPECT_RATIO_THRESHOLD && - ratio <= target_ratio) { - should_pass = true; - } else if (filter_type == VideoFilteringCalculatorOptions::AspectRatioFilter:: - LOWER_ASPECT_RATIO_THRESHOLD && - ratio >= target_ratio) { - should_pass = true; - } - if (should_pass) { - cc->Outputs().Tag(kOutputFrameTag).AddPacket(input_packet); - return absl::OkStatus(); - } - if (options.fail_if_any()) { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << absl::Substitute( - "Failing due to aspect ratio. Target aspect ratio: $0. Frame " - "width: $1, height: $2.", - target_ratio, frame.Width(), frame.Height()); - } - - return absl::OkStatus(); -} -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto b/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto deleted file mode 100644 index 997696a0f..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator.proto +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/framework/calculator.proto"; - -message VideoFilteringCalculatorOptions { - extend mediapipe.CalculatorOptions { - optional VideoFilteringCalculatorOptions ext = 278504113; - } - - // If true, when an input frame needs be filtered out according to the filter - // type and conditions, the calculator would return a FAIL status. Otherwise, - // the calculator would simply skip the filtered frames and would not pass it - // down to downstream nodes. - optional bool fail_if_any = 1 [default = false]; - - message AspectRatioFilter { - // Target width and height, which define the aspect ratio - // (i.e. target_width / target_height) to compare input frames with. The - // actual values of these fields do not matter, only the ratio between them - // does. These values must be set to positive. - optional int32 target_width = 1 [default = -1]; - optional int32 target_height = 2 [default = -1]; - enum FilterType { - UNKNOWN_FILTER_TYPE = 0; - // Use this type when the target width and height defines an upper bound - // (inclusive) of the aspect ratio. - UPPER_ASPECT_RATIO_THRESHOLD = 1; - // Use this type when the target width and height defines a lower bound - // (inclusive) of the aspect ratio. - LOWER_ASPECT_RATIO_THRESHOLD = 2; - // Use this type to configure the calculator as a no-op pass-through node. - NO_FILTERING = 3; - } - optional FilterType filter_type = 3; - } - - oneof filter { - AspectRatioFilter aspect_ratio_filter = 2; - } -} diff --git a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc deleted file mode 100644 index 758193832..000000000 --- a/mediapipe/examples/desktop/autoflip/calculators/video_filtering_calculator_test.cc +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "absl/strings/string_view.h" -#include "absl/strings/substitute.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_runner.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_builder.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -// Default configuration of the calculator. -CalculatorGraphConfig::Node GetCalculatorNode( - const std::string& fail_if_any, const std::string& extra_options = "") { - return ParseTextProtoOrDie( - absl::Substitute(R"( - calculator: "VideoFilteringCalculator" - input_stream: "INPUT_FRAMES:frames" - output_stream: "OUTPUT_FRAMES:output_frames" - options: { - [mediapipe.autoflip.VideoFilteringCalculatorOptions.ext]: { - fail_if_any: $0 - $1 - } - } - )", - fail_if_any, extra_options)); -} - -TEST(VideoFilterCalculatorTest, UpperBoundNoPass) { - CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"( - aspect_ratio_filter { - target_width: 2 - target_height: 1 - filter_type: UPPER_ASPECT_RATIO_THRESHOLD - } - )"); - - auto runner = ::absl::make_unique(config); - const int kFixedWidth = 1000; - const double kAspectRatio = 5.0 / 1.0; - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kFixedWidth, - static_cast(kFixedWidth / kAspectRatio), 16); - runner->MutableInputs() - ->Tag("INPUT_FRAMES") - .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000))); - MP_ASSERT_OK(runner->Run()); - const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets; - EXPECT_TRUE(output_packet.empty()); -} - -TEST(VerticalFrameRemovalCalculatorTest, UpperBoundPass) { - CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"( - aspect_ratio_filter { - target_width: 2 - target_height: 1 - filter_type: UPPER_ASPECT_RATIO_THRESHOLD - } - )"); - - auto runner = ::absl::make_unique(config); - const int kWidth = 1000; - const double kAspectRatio = 1.0 / 5.0; - const double kHeight = static_cast(kWidth / kAspectRatio); - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, kWidth, kHeight, 16); - runner->MutableInputs() - ->Tag("INPUT_FRAMES") - .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000))); - MP_ASSERT_OK(runner->Run()); - const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets; - EXPECT_EQ(1, output_packet.size()); - auto& output_frame = output_packet[0].Get(); - EXPECT_EQ(kWidth, output_frame.Width()); - EXPECT_EQ(kHeight, output_frame.Height()); -} - -TEST(VideoFilterCalculatorTest, LowerBoundNoPass) { - CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"( - aspect_ratio_filter { - target_width: 2 - target_height: 1 - filter_type: LOWER_ASPECT_RATIO_THRESHOLD - } - )"); - - auto runner = ::absl::make_unique(config); - const int kFixedWidth = 1000; - const double kAspectRatio = 1.0 / 1.0; - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kFixedWidth, - static_cast(kFixedWidth / kAspectRatio), 16); - runner->MutableInputs() - ->Tag("INPUT_FRAMES") - .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000))); - MP_ASSERT_OK(runner->Run()); - const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets; - EXPECT_TRUE(output_packet.empty()); -} - -TEST(VerticalFrameRemovalCalculatorTest, LowerBoundPass) { - CalculatorGraphConfig::Node config = GetCalculatorNode("false", R"( - aspect_ratio_filter { - target_width: 2 - target_height: 1 - filter_type: LOWER_ASPECT_RATIO_THRESHOLD - } - )"); - - auto runner = ::absl::make_unique(config); - const int kWidth = 1000; - const double kAspectRatio = 5.0 / 1.0; - const double kHeight = static_cast(kWidth / kAspectRatio); - auto input_frame = - ::absl::make_unique(ImageFormat::SRGB, kWidth, kHeight, 16); - runner->MutableInputs() - ->Tag("INPUT_FRAMES") - .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000))); - MP_ASSERT_OK(runner->Run()); - const auto& output_packet = runner->Outputs().Tag("OUTPUT_FRAMES").packets; - EXPECT_EQ(1, output_packet.size()); - auto& output_frame = output_packet[0].Get(); - EXPECT_EQ(kWidth, output_frame.Width()); - EXPECT_EQ(kHeight, output_frame.Height()); -} - -// Test that an error should be generated when fail_if_any is true. -TEST(VerticalFrameRemovalCalculatorTest, OutputError) { - CalculatorGraphConfig::Node config = GetCalculatorNode("true", R"( - aspect_ratio_filter { - target_width: 2 - target_height: 1 - filter_type: LOWER_ASPECT_RATIO_THRESHOLD - } - )"); - - auto runner = ::absl::make_unique(config); - const int kFixedWidth = 1000; - const double kAspectRatio = 1.0 / 1.0; - auto input_frame = ::absl::make_unique( - ImageFormat::SRGB, kFixedWidth, - static_cast(kFixedWidth / kAspectRatio), 16); - runner->MutableInputs() - ->Tag("INPUT_FRAMES") - .packets.push_back(Adopt(input_frame.release()).At(Timestamp(1000))); - absl::Status status = runner->Run(); - EXPECT_EQ(status.code(), absl::StatusCode::kUnknown); - EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("Failing due to aspect ratio")); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/BUILD b/mediapipe/examples/desktop/autoflip/quality/BUILD deleted file mode 100644 index 307953c19..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/BUILD +++ /dev/null @@ -1,374 +0,0 @@ -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -proto_library( - name = "cropping_proto", - srcs = ["cropping.proto"], - deps = [ - "//mediapipe/examples/desktop/autoflip:autoflip_messages_proto", - "//mediapipe/examples/desktop/autoflip/quality:kinematic_path_solver_proto", - ], -) - -mediapipe_cc_proto_library( - name = "cropping_cc_proto", - srcs = ["cropping.proto"], - cc_deps = [ - ":kinematic_path_solver_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - ], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":cropping_proto"], -) - -proto_library( - name = "kinematic_path_solver_proto", - srcs = ["kinematic_path_solver.proto"], -) - -mediapipe_cc_proto_library( - name = "kinematic_path_solver_cc_proto", - srcs = ["kinematic_path_solver.proto"], - visibility = [ - "//mediapipe/examples:__subpackages__", - ], - deps = [":kinematic_path_solver_proto"], -) - -proto_library( - name = "focus_point_proto", - srcs = ["focus_point.proto"], -) - -mediapipe_cc_proto_library( - name = "focus_point_cc_proto", - srcs = ["focus_point.proto"], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":focus_point_proto"], -) - -cc_library( - name = "frame_crop_region_computer", - srcs = ["frame_crop_region_computer.cc"], - hdrs = ["frame_crop_region_computer.h"], - deps = [ - ":cropping_cc_proto", - ":utils", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "math_utils", - hdrs = ["math_utils.h"], -) - -cc_library( - name = "piecewise_linear_function", - srcs = ["piecewise_linear_function.cc"], - hdrs = ["piecewise_linear_function.h"], - deps = [ - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "padding_effect_generator", - srcs = ["padding_effect_generator.cc"], - hdrs = ["padding_effect_generator.h"], - deps = [ - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - ], -) - -cc_library( - name = "scene_camera_motion_analyzer", - srcs = ["scene_camera_motion_analyzer.cc"], - hdrs = ["scene_camera_motion_analyzer.h"], - deps = [ - ":cropping_cc_proto", - ":focus_point_cc_proto", - ":math_utils", - ":piecewise_linear_function", - ":utils", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:timestamp", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], -) - -cc_library( - name = "scene_cropping_viz", - srcs = ["scene_cropping_viz.cc"], - hdrs = ["scene_cropping_viz.h"], - deps = [ - ":cropping_cc_proto", - ":focus_point_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "polynomial_regression_path_solver", - srcs = ["polynomial_regression_path_solver.cc"], - hdrs = ["polynomial_regression_path_solver.h"], - deps = [ - ":focus_point_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@ceres_solver//:ceres", - ], -) - -cc_library( - name = "kinematic_path_solver", - srcs = ["kinematic_path_solver.cc"], - hdrs = ["kinematic_path_solver.h"], - deps = [ - ":kinematic_path_solver_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_test( - name = "kinematic_path_solver_test", - srcs = ["kinematic_path_solver_test.cc"], - deps = [ - ":kinematic_path_solver", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "scene_cropper", - srcs = ["scene_cropper.cc"], - hdrs = ["scene_cropper.h"], - deps = [ - ":cropping_cc_proto", - ":focus_point_cc_proto", - ":kinematic_path_solver", - ":polynomial_regression_path_solver", - ":utils", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "utils", - srcs = ["utils.cc"], - hdrs = ["utils.h"], - deps = [ - ":cropping_cc_proto", - ":math_utils", - ":piecewise_linear_function", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "frame_crop_region_computer_test", - srcs = ["frame_crop_region_computer_test.cc"], - deps = [ - ":cropping_cc_proto", - ":frame_crop_region_computer", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "piecewise_linear_function_test", - srcs = ["piecewise_linear_function_test.cc"], - deps = [ - ":piecewise_linear_function", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - ], -) - -cc_test( - name = "scene_camera_motion_analyzer_test", - srcs = ["scene_camera_motion_analyzer_test.cc"], - data = [ - "//mediapipe/examples/desktop/autoflip/quality/testdata:camera_motion_tracking_scene_frame_results.csv", - ], - deps = [ - ":focus_point_cc_proto", - ":piecewise_linear_function", - ":scene_camera_motion_analyzer", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "padding_effect_generator_test", - srcs = ["padding_effect_generator_test.cc"], - data = [ - "//mediapipe/examples/desktop/autoflip/quality/testdata:google.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.3_solid_background.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_0.6_solid_background.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.6_solid_background.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_1_solid_background.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_2.5_solid_background.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4.jpg", - "//mediapipe/examples/desktop/autoflip/quality/testdata:result_3.4_solid_background.jpg", - ], - deps = [ - ":padding_effect_generator", - "//mediapipe/framework/deps:file_path", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "polynomial_regression_path_solver_test", - srcs = ["polynomial_regression_path_solver_test.cc"], - deps = [ - ":focus_point_cc_proto", - ":polynomial_regression_path_solver", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:status", - ], -) - -cc_test( - name = "scene_cropper_test", - size = "small", - timeout = "short", - srcs = ["scene_cropper_test.cc"], - deps = [ - ":focus_point_cc_proto", - ":scene_cropper", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - ], -) - -cc_test( - name = "utils_test", - srcs = ["utils_test.cc"], - deps = [ - ":cropping_cc_proto", - ":utils", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:opencv_core", - ], -) - -proto_library( - name = "visual_scorer_proto", - srcs = ["visual_scorer.proto"], -) - -mediapipe_cc_proto_library( - name = "visual_scorer_cc_proto", - srcs = ["visual_scorer.proto"], - visibility = ["//mediapipe/examples:__subpackages__"], - deps = [":visual_scorer_proto"], -) - -cc_library( - name = "visual_scorer", - srcs = ["visual_scorer.cc"], - hdrs = ["visual_scorer.h"], - deps = [ - ":visual_scorer_cc_proto", - "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_test( - name = "visual_scorer_test", - srcs = [ - "visual_scorer_test.cc", - ], - linkstatic = 1, - deps = [ - ":visual_scorer", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) diff --git a/mediapipe/examples/desktop/autoflip/quality/cropping.proto b/mediapipe/examples/desktop/autoflip/quality/cropping.proto deleted file mode 100644 index f8a62b247..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/cropping.proto +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -import "mediapipe/examples/desktop/autoflip/autoflip_messages.proto"; -import "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto"; - -// All relevant information for key frames, including timestamp and detected -// features. This object should be generated by calling PackKeyFrameInfo() in -// the util namespace. It is passed in to ComputeFrameCropRegion(). -message KeyFrameInfo { - // Frame timestamp (in microseconds). - optional int64 timestamp_ms = 1; - // Detected features. - optional DetectionSet detections = 2; -} - -// User-specified key frame crop options (such as target width and height). -message KeyFrameCropOptions { - // Target crop size. - // Note: if you are using the SceneCroppingCalculator, DO NOT set these fields - // manually as they will be then overwritten inside the calculator. - optional int32 target_width = 1; - optional int32 target_height = 2; - // Option for how region score is aggregated from individual feature scores. - // TODO: consider merging this enum type into the signal fusing - // calculator. - enum ScoreAggregationType { - // Unknown value (should not be used). - UNKNOWN = 0; - // Takes the score of the feature with maximum score. - MAXIMUM = 1; - // Takes the sum of the scores of the required regions. - SUM_REQUIRED = 2; - // Takes the sum of the scores of all the regions that are fully covered. - SUM_ALL = 3; - // Uses a constant score 1.0 for all crop regions. - CONSTANT = 4; - } - optional ScoreAggregationType score_aggregation_type = 3 [default = SUM_ALL]; - // Minimum centered coverage fraction (in length, not area) for a non-required - // region to be included in the crop region. Applies to both dimensions. - optional float non_required_region_min_coverage_fraction = 4 [default = 0.5]; -} - -// Key frame crop result containing the crop region rectangle, along with -// summary information on the cropping, such as whether all required regions -// could fit inside the target size, and what fraction of non-required regions -// are fully covered. This object is returned by ComputeFrameCropRegion() in -// the FrameCropRegionComputer class. -message KeyFrameCropResult { - // Successfully covers all required features. If there are no required - // regions, this field is set to true. - optional bool are_required_regions_covered_in_target_size = 1; - // Fraction of non-required features covered. - optional float fraction_non_required_covered = 2; - // Whether required crop region is empty (no detections). - optional bool required_region_is_empty = 3; - // Whether (full) crop region is empty (no detections). - optional bool region_is_empty = 4; - // Computed required crop region. - optional Rect required_region = 5; - // Computed (full) crop region. - optional Rect region = 6; - // Score of the computed crop region based on the detected features. - optional float region_score = 7; - // Frame timestamp (in microseconds). - optional int64 timestamp_ms = 8; -} - -// Compact processed scene key frame info containing timestamp, center position, -// and score. Each key frame has one SceneKeyFrameCompactInfo in -// SceneKeyFrameCropSummary. -message SceneKeyFrameCompactInfo { - // Key frame timestamp (in microseconds). - optional int64 timestamp_ms = 1; - // Key frame crop region center in the horizontal/vertical directions (in - // pixels). - optional float center_x = 2; - optional float center_y = 3; - // Key frame crop region score. - optional float score = 4; -} - -// Summary information for the key frame crop results in a scene. Computed by -// AnalyzeSceneKeyFrameCropResults() in the SceneCameraMotionAnalyzer class. -// Used to decide camera motion type and populate salient point frames. -message SceneKeyFrameCropSummary { - // Scene frame size. - optional int32 scene_frame_width = 1; - optional int32 scene_frame_height = 2; - - // Number of key frames in the scene. - optional int32 num_key_frames = 3; - // Scene key frame compact infos. - repeated SceneKeyFrameCompactInfo key_frame_compact_infos = 4; - - // The minimum/maximum values of key frames' crop centers in the horizontal/ - // vertical directions. - optional float key_frame_center_min_x = 5; - optional float key_frame_center_max_x = 6; - optional float key_frame_center_min_y = 7; - optional float key_frame_center_max_y = 8; - - // The union of all the key frame required crop regions. When camera is steady - // the crop window is set to cover this union. - optional Rect key_frame_required_crop_region_union = 9; - - // The minimum/maximum scores of key frames' crop regions. - optional float key_frame_min_score = 10; - optional float key_frame_max_score = 11; - - // Size of the scene's crop window, calculated as the maximum of the target - // size and the largest size of the key frames' crop regions in the scene. - optional int32 crop_window_width = 12; - optional int32 crop_window_height = 13; - - // Indicator for whether the scene has any frame with any salient region. - optional bool has_salient_region = 14; - // Indicator for whether the scene has any frame with any required salient - // region. - optional bool has_required_salient_region = 15; - // Percentage of key frames that are successfully cropped (i.e. covers all - // required regions inside the target size). - optional float frame_success_rate = 16; - // Amount of motion in the horizontal/vertical direction (i.e. the horizontal/ - // vertical range of the key frame crop centers' position as a fraction of - // frame width/height). - optional float horizontal_motion_amount = 17; - optional float vertical_motion_amount = 18; -} - -// Scene camera motion determined by the SceneCameraMotionAnalyzer class. -message SceneCameraMotion { - // Camera focuses on a fixed center throughout the scene. - message SteadyMotion { - // Steady look-at center in horizontal/vertical directions (in pixels). - optional float steady_look_at_center_x = 1; - optional float steady_look_at_center_y = 2; - } - // Camera tracks key frame salient region centers. - message TrackingMotion { - // Fields to be added if necessary. - } - // Camera sweeps from one point to another. - message SweepingMotion { - // Starting and ending center positions for camera sweeping in pixels. - optional float sweep_start_center_x = 1; - optional float sweep_start_center_y = 2; - optional float sweep_end_center_x = 3; - optional float sweep_end_center_y = 4; - } - oneof motion_type { - SteadyMotion steady_motion = 1; - TrackingMotion tracking_motion = 2; - SweepingMotion sweeping_motion = 3; - // Other types that we might support later. - } -} - -// User-specified options for analyzing scene camera motion from a collection of -// key frame crop regions. -message SceneCameraMotionAnalyzerOptions { - reserved 9; - // If there is small motion within the scene keep the camera steady at the - // center. - optional float motion_stabilization_threshold_percent = 1 [default = .30]; - // Snap to center if there is small motion and already focused closed to the - // center. - optional float snap_center_max_distance_percent = 2 [default = .08]; - // Maximum weight for a constraint. Scales scores accordingly so that the - // maximum score is equal to this weight. - optional float maximum_salient_point_weight = 3 [default = 100.0]; - // Normalized bound for SalientPoint's in the frame from the border. This is - // uniformly applied to the left, right, top, and bottom. It should be - // strictly less than 0.5. A narrower bound (closer to 0.5) gives better - // constraint enforcement. - optional float salient_point_bound = 4 [default = 0.48]; - // Indicator for whether sweeping is allowed. Note that if a scene can be - // seamlessly padded with solid background color, sweeping will be disabled - // regardlessly of the value of this flag. - optional bool allow_sweeping = 5 [default = true]; - // Minimal scene time span in seconds to allow camera sweeping. - optional float minimum_scene_span_sec_for_sweeping = 6 [default = 1.0]; - // If success rate in a scene is less than this, then use camera sweeping. - optional float minimum_success_rate_for_sweeping = 7 [default = 0.4]; - // If true, sweep entire frame. Otherwise, sweep the crop window. - optional bool sweep_entire_frame = 8 [default = true]; - // When no salient region is received, the default behavior is the return the - // camera to center-focused location. When this flag is set to a value >0, - // the camera will remain at its last position for this amount of time before - // recentering (if the last scene camera motion type was steady). - optional int64 duration_before_centering_us = 10; -} - -// Video cropping summary information for debugging/statistics. -message VideoCroppingSummary { - message SceneCroppingSummary { - // Scene span in seconds. - optional float start_sec = 1; - optional float end_sec = 2; - // Indicator for whether this scene was cut at a real physical scene - // boundary (as opposed to force flush). - optional bool is_end_of_scene = 3; - // Scene camera motion. - optional SceneCameraMotion camera_motion = 4; - // Indicator for whether the scene is padded. - optional bool is_padded = 5; - } - // Cropping summaries for all the scenes in the video. - repeated SceneCroppingSummary scene_summaries = 1; -} - -message CameraMotionOptions { - message PolynomialRegressionPathSolver { - // Number of frames from prior buffer to be used to smooth out camera - // trajectory when it was a forced flush. - optional int32 prior_frame_buffer_size = 1 [default = 30]; - } - oneof camera_model_oneof { - // Fits a poly line to keypoints to find a smooth camera path. - PolynomialRegressionPathSolver polynomial_path_solver = 1; - // Maintains a kinematic state of the camera, updated with keypoints, to - // find a smooth camera path. Currently optimized for real-time operation. - KinematicOptions kinematic_options = 2; - } -} diff --git a/mediapipe/examples/desktop/autoflip/quality/focus_point.proto b/mediapipe/examples/desktop/autoflip/quality/focus_point.proto deleted file mode 100644 index e53b3ed0a..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/focus_point.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -// Focus point location (normalized w.r.t. frame_width and frame_height, i.e. -// specified in the domain [0, 1] x [0, 1]). - -// For TYPE_INCLUDE: -// During retargeting and stabilization focus points introduce constraints -// that will try to keep the normalized location in the rectangle -// frame_size - normalized bounds. -// For this soft constraints are used, therefore the weight specifies -// how "important" the focus point is (higher is better). -// In particular for each point p the retargeter introduces two pairs of -// constraints of the form: -// x - slack < width - right -// and x + slack > 0 + left, with slack > 0 -// where the weight specifies the importance of the slack. -// -// For TYPE_EXCLUDE_*: -// Similar to above, but constraints are introduced to keep -// the point to the left of the left bound OR the right of the right bound. -// In particular: -// x - slack < left OR -// x + slack >= right -// Similar to above, the weight specifies the importance of the slack. -// -// Note: Choosing a too high weight can lead to -// jerkiness as the stabilization essentially starts tracking the focus point. -message FocusPoint { - // Normalized location of the point (within domain [0, 1] x [0, 1]. - optional float norm_point_x = 1 [default = 0.0]; - optional float norm_point_y = 2 [default = 0.0]; - - enum FocusPointType { - TYPE_INCLUDE = 1; - TYPE_EXCLUDE_LEFT = 2; - TYPE_EXCLUDE_RIGHT = 3; - } - - // Focus point type. By default we try to frame the focus point within - // the bounding box specified by left, bottom, right, top. Alternatively, one - // can choose to exclude the point. For details, see discussion above. - optional FocusPointType type = 11 [default = TYPE_INCLUDE]; - - // Bounds are specified in normalized coordinates [0, 1], FROM the specified - // border. Opposing bounds (e.g. left and right) may not add to values - // larger than 1. - // Default bounds center focus point within centering third of the frame. - optional float left = 3 [default = 0.3]; - optional float bottom = 4 [default = 0.3]; - optional float right = 9 [default = 0.3]; - optional float top = 10 [default = 0.3]; - - optional float weight = 5 [default = 15]; - - extensions 20000 to max; -} - -// Aggregates FocusPoint's for a frame. -message FocusPointFrame { - repeated FocusPoint point = 1; - - extensions 20000 to max; -} diff --git a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc b/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc deleted file mode 100644 index 5916d1829..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h" - -#include - -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace autoflip { - -absl::Status FrameCropRegionComputer::ExpandSegmentUnderConstraint( - const Segment& segment_to_add, const Segment& base_segment, - const int max_length, Segment* combined_segment, - CoverType* cover_type) const { - RET_CHECK(combined_segment != nullptr) << "Combined segment is null."; - RET_CHECK(cover_type != nullptr) << "Cover type is null."; - - const LeftPoint segment_to_add_left = segment_to_add.first; - const RightPoint segment_to_add_right = segment_to_add.second; - RET_CHECK(segment_to_add_right >= segment_to_add_left) - << "Invalid segment to add."; - const LeftPoint base_segment_left = base_segment.first; - const RightPoint base_segment_right = base_segment.second; - RET_CHECK(base_segment_right >= base_segment_left) << "Invalid base segment."; - const int base_length = base_segment_right - base_segment_left; - RET_CHECK(base_length <= max_length) - << "Base segment length exceeds max length."; - - const int segment_to_add_length = segment_to_add_right - segment_to_add_left; - const int max_leftout_amount = - std::ceil((1.0 - options_.non_required_region_min_coverage_fraction()) * - segment_to_add_length / 2); - const LeftPoint min_coverage_segment_to_add_left = - segment_to_add_left + max_leftout_amount; - const LeftPoint min_coverage_segment_to_add_right = - segment_to_add_right - max_leftout_amount; - - LeftPoint combined_segment_left = - std::min(segment_to_add_left, base_segment_left); - RightPoint combined_segment_right = - std::max(segment_to_add_right, base_segment_right); - - LeftPoint min_coverage_combined_segment_left = - std::min(min_coverage_segment_to_add_left, base_segment_left); - RightPoint min_coverage_combined_segment_right = - std::max(min_coverage_segment_to_add_right, base_segment_right); - - if ((combined_segment_right - combined_segment_left) <= max_length) { - *cover_type = FULLY_COVERED; - } else if (min_coverage_combined_segment_right - - min_coverage_combined_segment_left <= - max_length) { - *cover_type = PARTIALLY_COVERED; - combined_segment_left = min_coverage_combined_segment_left; - combined_segment_right = min_coverage_combined_segment_right; - } else { - *cover_type = NOT_COVERED; - combined_segment_left = base_segment_left; - combined_segment_right = base_segment_right; - } - - *combined_segment = - std::make_pair(combined_segment_left, combined_segment_right); - return absl::OkStatus(); -} - -absl::Status FrameCropRegionComputer::ExpandRectUnderConstraints( - const Rect& rect_to_add, const int max_width, const int max_height, - Rect* base_rect, CoverType* cover_type) const { - RET_CHECK(base_rect != nullptr) << "Base rect is null."; - RET_CHECK(cover_type != nullptr) << "Cover type is null."; - RET_CHECK(base_rect->width() <= max_width && - base_rect->height() <= max_height) - << "Base rect already exceeds target size."; - - const LeftPoint rect_to_add_left = rect_to_add.x(); - const RightPoint rect_to_add_right = rect_to_add.x() + rect_to_add.width(); - const LeftPoint rect_to_add_top = rect_to_add.y(); - const RightPoint rect_to_add_bottom = rect_to_add.y() + rect_to_add.height(); - const LeftPoint base_rect_left = base_rect->x(); - const RightPoint base_rect_right = base_rect->x() + base_rect->width(); - const LeftPoint base_rect_top = base_rect->y(); - const RightPoint base_rect_bottom = base_rect->y() + base_rect->height(); - - Segment horizontal_combined_segment, vertical_combined_segment; - CoverType horizontal_cover_type, vertical_cover_type; - const auto horizontal_status = ExpandSegmentUnderConstraint( - std::make_pair(rect_to_add_left, rect_to_add_right), - std::make_pair(base_rect_left, base_rect_right), max_width, - &horizontal_combined_segment, &horizontal_cover_type); - MP_RETURN_IF_ERROR(horizontal_status); - const auto vertical_status = ExpandSegmentUnderConstraint( - std::make_pair(rect_to_add_top, rect_to_add_bottom), - std::make_pair(base_rect_top, base_rect_bottom), max_height, - &vertical_combined_segment, &vertical_cover_type); - MP_RETURN_IF_ERROR(vertical_status); - - if (horizontal_cover_type == NOT_COVERED || - vertical_cover_type == NOT_COVERED) { - // Gives up if the segment is not covered in either direction. - *cover_type = NOT_COVERED; - } else { - // Tries to (partially) cover the new rect to be added. - base_rect->set_x(horizontal_combined_segment.first); - base_rect->set_y(vertical_combined_segment.first); - base_rect->set_width(horizontal_combined_segment.second - - horizontal_combined_segment.first); - base_rect->set_height(vertical_combined_segment.second - - vertical_combined_segment.first); - if (horizontal_cover_type == FULLY_COVERED && - vertical_cover_type == FULLY_COVERED) { - *cover_type = FULLY_COVERED; - } else { - *cover_type = PARTIALLY_COVERED; - } - } - - return absl::OkStatus(); -} - -void FrameCropRegionComputer::UpdateCropRegionScore( - const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type, - const float feature_score, const bool is_required, - float* crop_region_score) { - if (feature_score < 0.0) { - LOG(WARNING) << "Ignoring negative score"; - return; - } - - switch (score_aggregation_type) { - case KeyFrameCropOptions::MAXIMUM: { - *crop_region_score = std::max(feature_score, *crop_region_score); - break; - } - case KeyFrameCropOptions::SUM_REQUIRED: { - if (is_required) { - *crop_region_score += feature_score; - } - break; - } - case KeyFrameCropOptions::SUM_ALL: { - *crop_region_score += feature_score; - break; - } - case KeyFrameCropOptions::CONSTANT: { - *crop_region_score = 1.0; - break; - } - default: { - LOG(WARNING) << "Unknown CropRegionScoreType " << score_aggregation_type; - break; - } - } -} - -absl::Status FrameCropRegionComputer::ComputeFrameCropRegion( - const KeyFrameInfo& frame_info, KeyFrameCropResult* crop_result) const { - RET_CHECK(crop_result != nullptr) << "KeyFrameCropResult is null."; - - // Set timestamp of KeyFrameCropResult - crop_result->set_timestamp_ms(frame_info.timestamp_ms()); - - // Sorts required and non-required regions. - std::vector required_regions, non_required_regions; - const auto sort_status = SortDetections( - frame_info.detections(), &required_regions, &non_required_regions); - MP_RETURN_IF_ERROR(sort_status); - - int target_width = options_.target_width(); - int target_height = options_.target_height(); - auto* region = crop_result->mutable_region(); - - bool crop_region_is_empty = true; - float crop_region_score = 0.0; - - // Gets union of all required regions. - for (int i = 0; i < required_regions.size(); ++i) { - const Rect& required_region = required_regions[i].location(); - if (crop_region_is_empty) { - *region = required_region; - crop_region_is_empty = false; - } else { - RectUnion(required_region, region); - } - UpdateCropRegionScore(options_.score_aggregation_type(), - required_regions[i].score(), true, - &crop_region_score); - } - crop_result->set_required_region_is_empty(crop_region_is_empty); - if (!crop_region_is_empty) { - *crop_result->mutable_required_region() = *region; - crop_result->set_are_required_regions_covered_in_target_size( - region->width() <= target_width && region->height() <= target_height); - target_width = std::max(target_width, region->width()); - target_height = std::max(target_height, region->height()); - } else { - crop_result->set_are_required_regions_covered_in_target_size(true); - } - - // Tries to fit non-required regions. - int num_covered = 0; - for (int i = 0; i < non_required_regions.size(); ++i) { - const Rect& non_required_region = non_required_regions[i].location(); - CoverType cover_type = NOT_COVERED; - if (crop_region_is_empty) { - // If the crop region is empty, tries to expand an empty base region - // at the center of this region to include itself. - region->set_x(non_required_region.x() + non_required_region.width() / 2); - region->set_y(non_required_region.y() + non_required_region.height() / 2); - region->set_width(0); - region->set_height(0); - MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region, - target_width, target_height, - region, &cover_type)); - if (cover_type != NOT_COVERED) { - crop_region_is_empty = false; - } - } else { - // Otherwise tries to expand the crop region to cover the non-required - // region under target size constraint. - MP_RETURN_IF_ERROR(ExpandRectUnderConstraints(non_required_region, - target_width, target_height, - region, &cover_type)); - } - - // Updates number of covered non-required regions and score. - if (cover_type == FULLY_COVERED) { - num_covered++; - UpdateCropRegionScore(options_.score_aggregation_type(), - non_required_regions[i].score(), false, - &crop_region_score); - } - } - - const float fraction_covered = - non_required_regions.empty() - ? 0.0 - : static_cast(num_covered) / non_required_regions.size(); - crop_result->set_fraction_non_required_covered(fraction_covered); - - crop_result->set_region_is_empty(crop_region_is_empty); - crop_result->set_region_score(crop_region_score); - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h b/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h deleted file mode 100644 index b2be9e28c..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// This class computes per-frame crop regions based on crop frame options. -// It aggregates required regions and then tries to fit in non-required regions -// with best effort. It does not make use of static features. -class FrameCropRegionComputer { - public: - FrameCropRegionComputer() = delete; - - explicit FrameCropRegionComputer( - const KeyFrameCropOptions& crop_frame_options) - : options_(crop_frame_options) {} - - ~FrameCropRegionComputer() {} - - // Computes the crop region for the key frame using the crop options. The crop - // region covers all the required regions, and attempts to cover the - // non-required regions with best effort. Note: this function does not - // consider static features, and simply tries to fit the detected features - // within the target frame size. The score of the crop region is aggregated - // from individual feature scores given the score aggregation type. - absl::Status ComputeFrameCropRegion(const KeyFrameInfo& frame_info, - KeyFrameCropResult* crop_result) const; - - protected: - // A segment is a 1-d object defined by its left and right point. - using LeftPoint = int; - using RightPoint = int; - using Segment = std::pair; - // How much a segment is covered in the combined segment. - enum CoverType { - FULLY_COVERED = 1, - PARTIALLY_COVERED = 2, - NOT_COVERED = 3, - }; - // Expands a base segment to cover a segment to be added given maximum length - // constraint. The operation is best-effort. The resulting enlarged segment is - // set in the returned combined segment. Returns a CoverType to indicate the - // coverage of the segment to be added in the combined segment. - // There are 3 cases: - // case 1: the length of the union of the two segments is not larger than - // the maximum length. - // In this case the combined segment is simply the union, and cover - // type is FULLY_COVERED. - // case 2: the union of the two segments exceeds the maximum length, but the - // union of the base segment and required minimum centered fraction - // of the new segment fits in the maximum length. - // In this case the combined segment is this latter union, and cover - // type is PARTIALLY_COVERED. - // case 3: the union of the base segment and required minimum centered - // fraction of the new segment exceeds the maximum length. - // In this case the combined segment is the base segment, and cover - // type is NOT_COVERED. - absl::Status ExpandSegmentUnderConstraint(const Segment& segment_to_add, - const Segment& base_segment, - const int max_length, - Segment* combined_segment, - CoverType* cover_type) const; - - // Expands a base rectangle to cover a new rectangle to be added under width - // and height constraints. The operation is best-effort. It considers - // horizontal and vertical directions separately, using the - // ExpandSegmentUnderConstraint function for each direction. The cover type is - // FULLY_COVERED if the new rectangle is fully covered in both directions, - // PARTIALLY_COVERED if it is at least partially covered in both directions, - // and NOT_COVERED if it is not covered in either direction. - absl::Status ExpandRectUnderConstraints(const Rect& rect_to_add, - const int max_width, - const int max_height, Rect* base_rect, - CoverType* cover_type) const; - - // Updates crop region score given current feature score, whether the feature - // is required, and the score aggregation type. Ignores negative scores. - static void UpdateCropRegionScore( - const KeyFrameCropOptions::ScoreAggregationType score_aggregation_type, - const float feature_score, const bool is_required, - float* crop_region_score); - - private: - // Crop frame options. - KeyFrameCropOptions options_; -}; -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_FRAME_CROP_REGION_COMPUTER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc b/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc deleted file mode 100644 index eb60ea64e..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer_test.cc +++ /dev/null @@ -1,579 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.h" - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { - -using ::testing::HasSubstr; - -const int kSegmentMaxLength = 10; -const int kTargetWidth = 500; -const int kTargetHeight = 1000; - -// Makes a rectangle given the corner (x, y) and the size (width, height). -Rect MakeRect(const int x, const int y, const int width, const int height) { - Rect rect; - rect.set_x(x); - rect.set_y(y); - rect.set_width(width); - rect.set_height(height); - return rect; -} - -// Adds a detection to the key frame info given its location, whether it is -// required, and its score. The score is default to 1.0. -void AddDetection(const Rect& rect, const bool is_required, - KeyFrameInfo* key_frame_info, const float score = 1.0) { - auto* detection = key_frame_info->mutable_detections()->add_detections(); - *(detection->mutable_location()) = rect; - detection->set_score(score); - detection->set_is_required(is_required); -} - -// Makes key frame crop options given target width and height. -KeyFrameCropOptions MakeKeyFrameCropOptions(const int target_width, - const int target_height) { - KeyFrameCropOptions options; - options.set_target_width(target_width); - options.set_target_height(target_height); - return options; -} - -// Checks whether rectangle a is inside rectangle b. -bool CheckRectIsInside(const Rect& rect_a, const Rect& rect_b) { - return (rect_b.x() <= rect_a.x() && rect_b.y() <= rect_a.y() && - rect_b.x() + rect_b.width() >= rect_a.x() + rect_a.width() && - rect_b.y() + rect_b.height() >= rect_a.y() + rect_a.height()); -} - -// Checks whether two rectangles are equal. -bool CheckRectsEqual(const Rect& rect1, const Rect& rect2) { - return (rect1.x() == rect2.x() && rect1.y() == rect2.y() && - rect1.width() == rect2.width() && rect1.height() == rect2.height()); -} - -// Checks whether two rectangles have non-zero overlapping area. -bool CheckRectsOverlap(const Rect& rect1, const Rect& rect2) { - const int x1_left = rect1.x(), x1_right = rect1.x() + rect1.width(); - const int y1_top = rect1.y(), y1_bottom = rect1.y() + rect1.height(); - const int x2_left = rect2.x(), x2_right = rect2.x() + rect2.width(); - const int y2_top = rect2.y(), y2_bottom = rect2.y() + rect2.height(); - const int x_left = std::max(x1_left, x2_left); - const int x_right = std::min(x1_right, x2_right); - const int y_top = std::max(y1_top, y2_top); - const int y_bottom = std::min(y1_bottom, y2_bottom); - return (x_right > x_left && y_bottom > y_top); -} - -// Checks that all the required regions in the detections in KeyFrameInfo are -// covered in the KeyFrameCropResult. -void CheckRequiredRegionsAreCovered(const KeyFrameInfo& key_frame_info, - const KeyFrameCropResult& result) { - bool has_required = false; - for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) { - const auto& detection = key_frame_info.detections().detections(i); - if (detection.is_required()) { - has_required = true; - EXPECT_TRUE( - CheckRectIsInside(detection.location(), result.required_region())); - } - } - EXPECT_EQ(has_required, !result.required_region_is_empty()); - if (has_required) { - EXPECT_FALSE(result.region_is_empty()); - EXPECT_TRUE(CheckRectIsInside(result.required_region(), result.region())); - } -} - -// Testable class that can access protected types and methods in the class. -class TestableFrameCropRegionComputer : public FrameCropRegionComputer { - public: - explicit TestableFrameCropRegionComputer(const KeyFrameCropOptions& options) - : FrameCropRegionComputer(options) {} - using FrameCropRegionComputer::CoverType; - using FrameCropRegionComputer::ExpandRectUnderConstraints; - using FrameCropRegionComputer::ExpandSegmentUnderConstraint; - using FrameCropRegionComputer::FULLY_COVERED; - using FrameCropRegionComputer::LeftPoint; // int - using FrameCropRegionComputer::NOT_COVERED; - using FrameCropRegionComputer::PARTIALLY_COVERED; - using FrameCropRegionComputer::RightPoint; // int - using FrameCropRegionComputer::Segment; // std::pair - using FrameCropRegionComputer::UpdateCropRegionScore; - - // Makes a segment from two endpoints. - static Segment MakeSegment(const LeftPoint left, const RightPoint right) { - return std::make_pair(left, right); - } - - // Checks that two segments are equal. - static bool CheckSegmentsEqual(const Segment& segment1, - const Segment& segment2) { - return (segment1.first == segment2.first && - segment1.second == segment2.second); - } -}; -using TestClass = TestableFrameCropRegionComputer; - -// Returns an instance of the testable class given -// non_required_region_min_coverage_fraction. -std::unique_ptr GetTestableClass( - const float non_required_region_min_coverage_fraction = 0.5) { - KeyFrameCropOptions options; - options.set_non_required_region_min_coverage_fraction( - non_required_region_min_coverage_fraction); - auto test_class = absl::make_unique(options); - return test_class; -} - -// Checks that ExpandSegmentUnderConstraint checks output pointers are not null. -TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckNull) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - TestClass::Segment base_segment = TestClass::MakeSegment(10, 15); - TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8); - TestClass::Segment combined_segment; - // Combined segment is null. - auto status = test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, nullptr, &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Combined segment is null.")); - // Cover type is null. - status = test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null.")); -} - -// Checks that ExpandSegmentUnderConstraint checks input segments are valid. -TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCheckValid) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - TestClass::Segment combined_segment; - - // Invalid base segment. - TestClass::Segment base_segment = TestClass::MakeSegment(15, 10); - TestClass::Segment segment_to_add = TestClass::MakeSegment(5, 8); - auto status = test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Invalid base segment.")); - - // Invalid segment to add. - base_segment = TestClass::MakeSegment(10, 15); - segment_to_add = TestClass::MakeSegment(8, 5); - status = test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Invalid segment to add.")); - - // Base segment exceeds max length. - base_segment = TestClass::MakeSegment(10, 100); - segment_to_add = TestClass::MakeSegment(5, 8); - status = test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Base segment length exceeds max length.")); -} - -// Checks that ExpandSegmentUnderConstraint handles case 1 properly: the length -// of the union of the two segments is not larger than the maximum length. -TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase1) { - auto test_class = GetTestableClass(); - TestClass::Segment combined_segment; - TestClass::CoverType cover_type; - TestClass::Segment base_segment = TestClass::MakeSegment(5, 10); - TestClass::Segment segment_to_add = TestClass::MakeSegment(3, 8); - MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type)); - EXPECT_EQ(cover_type, TestClass::FULLY_COVERED); - EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment, - TestClass::MakeSegment(3, 10))); -} - -// Checks that ExpandSegmentUnderConstraint handles case 2 properly: the union -// of the two segments exceeds the maximum length, but the union of the base -// segment with the minimum coverage fraction of the new segment is within the -// maximum length. -TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase2) { - TestClass::Segment combined_segment; - TestClass::CoverType cover_type; - TestClass::Segment base_segment = TestClass::MakeSegment(4, 8); - TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 16); - auto test_class = GetTestableClass(); - MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type)); - EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED); - EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment, - TestClass::MakeSegment(4, 12))); -} - -// Checks that ExpandSegmentUnderConstraint handles case 3 properly: the union -// of the base segment with the minimum coverage fraction of the new segment -// exceeds the maximum length. -TEST(FrameCropRegionComputerTest, ExpandSegmentUnderConstraintCase3) { - TestClass::Segment combined_segment; - TestClass::CoverType cover_type; - auto test_class = GetTestableClass(); - TestClass::Segment base_segment = TestClass::MakeSegment(6, 14); - TestClass::Segment segment_to_add = TestClass::MakeSegment(0, 4); - MP_EXPECT_OK(test_class->ExpandSegmentUnderConstraint( - segment_to_add, base_segment, kSegmentMaxLength, &combined_segment, - &cover_type)); - EXPECT_EQ(cover_type, TestClass::NOT_COVERED); - EXPECT_TRUE(TestClass::CheckSegmentsEqual(combined_segment, base_segment)); -} - -// Checks that ExpandRectUnderConstraints checks output pointers are not null. -TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksNotNull) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - Rect base_rect, rect_to_add; - // Base rect is null. - auto status = test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, nullptr, &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Base rect is null.")); - // Cover type is null. - status = test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Cover type is null.")); -} - -// Checks that ExpandRectUnderConstraints checks base rect is valid. -TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsChecksBaseValid) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - Rect base_rect = MakeRect(0, 0, 2 * kTargetWidth, 2 * kTargetHeight); - Rect rect_to_add; - const auto status = test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Base rect already exceeds target size.")); -} - -// Checks that ExpandRectUnderConstraints properly handles the case where the -// rectangle to be added can be fully covered. -TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsFullyCovered) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - Rect base_rect = MakeRect(0, 0, 50, 50); - Rect rect_to_add = MakeRect(30, 30, 30, 30); - MP_EXPECT_OK(test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type)); - EXPECT_EQ(cover_type, TestClass::FULLY_COVERED); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 60, 60))); -} - -// Checks that ExpandRectUnderConstraints properly handles the case where the -// rectangle to be added can be partially covered. -TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsPartiallyCovered) { - auto test_class = GetTestableClass(); - TestClass::CoverType cover_type; - // Rectangle to be added can be partially covered in both both dimensions. - Rect base_rect = MakeRect(0, 0, 500, 500); - Rect rect_to_add = MakeRect(0, 300, 600, 900); - MP_EXPECT_OK(test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type)); - EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975))); - - // Rectangle to be added can be fully covered in one dimension and partially - // covered in the other dimension. - base_rect = MakeRect(0, 0, 400, 500); - rect_to_add = MakeRect(100, 300, 400, 900); - MP_EXPECT_OK(test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type)); - EXPECT_EQ(cover_type, TestClass::PARTIALLY_COVERED); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 975))); -} - -// Checks that ExpandRectUnderConstraints properly handles the case where the -// rectangle to be added cannot be covered. -TEST(FrameCropRegionComputerTest, ExpandRectUnderConstraintsNotCovered) { - TestClass::CoverType cover_type; - auto test_class = GetTestableClass(); - Rect base_rect = MakeRect(0, 0, 500, 500); - Rect rect_to_add = MakeRect(550, 300, 100, 900); - MP_EXPECT_OK(test_class->ExpandRectUnderConstraints( - rect_to_add, kTargetWidth, kTargetHeight, &base_rect, &cover_type)); - EXPECT_EQ(cover_type, TestClass::NOT_COVERED); // no overlap in x dimension - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(0, 0, 500, 500))); -} - -// Checks that ComputeFrameCropRegion handles the case of empty detections. -TEST(FrameCropRegionComputerTest, HandlesEmptyDetections) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_TRUE(crop_result.region_is_empty()); -} - -// Checks that ComputeFrameCropRegion covers required regions when their union -// is within target size. -TEST(FrameCropRegionComputerTest, CoversRequiredWithinTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(100, 100, 100, 200), true, &key_frame_info); - AddDetection(MakeRect(200, 400, 300, 500), true, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(100, 100, 400, 800), - crop_result.required_region())); - EXPECT_TRUE( - CheckRectsEqual(crop_result.region(), crop_result.required_region())); - EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size()); -} - -// Checks that ComputeFrameCropRegion covers required regions when their union -// exceeds target size. -TEST(FrameCropRegionComputerTest, CoversRequiredExceedingTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info); - AddDetection(MakeRect(200, 400, 500, 500), true, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 700, 900), crop_result.region())); - EXPECT_TRUE( - CheckRectsEqual(crop_result.region(), crop_result.required_region())); - EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size()); -} - -// Checks that ComputeFrameCropRegion handles the case of only non-required -// regions and the region fits in the target size. -TEST(FrameCropRegionComputerTest, - HandlesOnlyNonRequiedRegionsInsideTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_TRUE(crop_result.required_region_is_empty()); - EXPECT_FALSE(crop_result.region_is_empty()); - EXPECT_TRUE( - CheckRectsEqual(key_frame_info.detections().detections(0).location(), - crop_result.region())); -} - -// Checks that ComputeFrameCropRegion handles the case of only non-required -// regions and the region exceeds the target size. -TEST(FrameCropRegionComputerTest, - HandlesOnlyNonRequiedRegionsExceedingTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(300, 600, 700, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_TRUE(crop_result.required_region_is_empty()); - EXPECT_FALSE(crop_result.region_is_empty()); - EXPECT_TRUE( - CheckRectsEqual(MakeRect(475, 600, 350, 100), crop_result.region())); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0); - EXPECT_TRUE( - CheckRectIsInside(crop_result.region(), - key_frame_info.detections().detections(0).location())); -} - -// Checks that ComputeFrameCropRegion covers non-required regions when their -// union fits within target size. -TEST(FrameCropRegionComputerTest, CoversNonRequiredInsideTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 100, 500), true, &key_frame_info); - AddDetection(MakeRect(300, 600, 100, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 700), crop_result.region())); - EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size()); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0); - for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) { - EXPECT_TRUE( - CheckRectIsInside(key_frame_info.detections().detections(i).location(), - crop_result.region())); - } -} - -// Checks that ComputeFrameCropRegion does not cover non-required regions that -// are outside the target size. -TEST(FrameCropRegionComputerTest, DoesNotCoverNonRequiredExceedingTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 500, 1000), true, &key_frame_info); - AddDetection(MakeRect(500, 0, 100, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 500, 1000), crop_result.region())); - EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size()); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0); - EXPECT_FALSE( - CheckRectIsInside(key_frame_info.detections().detections(1).location(), - crop_result.region())); -} - -// Checks that ComputeFrameCropRegion partially covers non-required regions that -// can partially fit in the target size. -TEST(FrameCropRegionComputerTest, - PartiallyCoversNonRequiredContainingTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(100, 0, 350, 1000), true, &key_frame_info); - AddDetection(MakeRect(0, 0, 650, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE( - CheckRectsEqual(MakeRect(100, 0, 387, 1000), crop_result.region())); - EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size()); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0); - EXPECT_TRUE( - CheckRectsOverlap(key_frame_info.detections().detections(1).location(), - crop_result.region())); -} - -// Checks that ComputeFrameCropRegion covers non-required regions when the -// required regions exceed target size. -TEST(FrameCropRegionComputerTest, - CoversNonRequiredWhenRequiredExceedsTargetSize) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 600, 1000), true, &key_frame_info); - AddDetection(MakeRect(450, 0, 100, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 600, 1000), crop_result.region())); - EXPECT_FALSE(crop_result.are_required_regions_covered_in_target_size()); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 1.0); - for (int i = 0; i < key_frame_info.detections().detections_size(); ++i) { - EXPECT_TRUE( - CheckRectIsInside(key_frame_info.detections().detections(i).location(), - crop_result.region())); - } -} - -// Checks that ComputeFrameCropRegion does not extend the crop region when -// the non-required region is too far. -TEST(FrameCropRegionComputerTest, - DoesNotExtendRegionWhenNonRequiredRegionIsTooFar) { - const auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info); - AddDetection(MakeRect(600, 0, 100, 100), false, &key_frame_info); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - CheckRequiredRegionsAreCovered(key_frame_info, crop_result); - EXPECT_TRUE(CheckRectsEqual(MakeRect(0, 0, 400, 400), crop_result.region())); - EXPECT_TRUE(crop_result.are_required_regions_covered_in_target_size()); - EXPECT_EQ(crop_result.fraction_non_required_covered(), 0.0); - EXPECT_FALSE( - CheckRectsOverlap(key_frame_info.detections().detections(1).location(), - crop_result.region())); -} - -// Checks that ComputeFrameCropRegion computes the score correctly when the -// aggregation type is maximum. -TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsMaximum) { - auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - options.set_score_aggregation_type(KeyFrameCropOptions::MAXIMUM); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1); - AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_FLOAT_EQ(crop_result.region_score(), 0.9f); -} - -// Checks that ComputeFrameCropRegion computes the score correctly when the -// aggregation type is sum required regions. -TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumRequired) { - auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - options.set_score_aggregation_type(KeyFrameCropOptions::SUM_REQUIRED); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1); - AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9); - AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f); -} - -// Checks that ComputeFrameCropRegion computes the score correctly when the -// aggregation type is sum all covered regions. -TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsSumAll) { - auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - options.set_score_aggregation_type(KeyFrameCropOptions::SUM_ALL); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1); - AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9); - AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_FLOAT_EQ(crop_result.region_score(), 1.5f); -} - -// Checks that ComputeFrameCropRegion computes the score correctly when the -// aggregation type is constant. -TEST(FrameCropRegionComputerTest, ComputesScoreWhenAggregationIsConstant) { - auto options = MakeKeyFrameCropOptions(kTargetWidth, kTargetHeight); - options.set_score_aggregation_type(KeyFrameCropOptions::CONSTANT); - FrameCropRegionComputer computer(options); - KeyFrameInfo key_frame_info; - AddDetection(MakeRect(0, 0, 400, 400), true, &key_frame_info, 0.1); - AddDetection(MakeRect(300, 300, 200, 500), true, &key_frame_info, 0.9); - AddDetection(MakeRect(300, 300, 200, 500), false, &key_frame_info, 0.5); - KeyFrameCropResult crop_result; - MP_EXPECT_OK(computer.ComputeFrameCropRegion(key_frame_info, &crop_result)); - EXPECT_FLOAT_EQ(crop_result.region_score(), 1.0f); -} -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc deleted file mode 100644 index c3d043273..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc +++ /dev/null @@ -1,248 +0,0 @@ -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h" - -namespace mediapipe { -namespace autoflip { -namespace { -int Median(const std::deque>& positions_raw) { - std::deque positions; - for (const auto& position : positions_raw) { - positions.push_back(position.second); - } - - size_t n = positions.size() / 2; - nth_element(positions.begin(), positions.begin() + n, positions.end()); - return positions[n]; -} -} // namespace -bool KinematicPathSolver::IsMotionTooSmall(double delta_degs) { - if (options_.has_min_motion_to_reframe()) { - return abs(delta_degs) < options_.min_motion_to_reframe(); - } else if (delta_degs > 0) { - return delta_degs < options_.min_motion_to_reframe_upper(); - } else { - return abs(delta_degs) < options_.min_motion_to_reframe_lower(); - } -} -void KinematicPathSolver::ClearHistory() { raw_positions_at_time_.clear(); } -absl::Status KinematicPathSolver::PredictMotionState(int position, - const uint64 time_us, - bool* state) { - if (!initialized_) { - *state = false; - return absl::OkStatus(); - } - - auto raw_positions_at_time_copy = raw_positions_at_time_; - - raw_positions_at_time_copy.push_front( - std::pair(time_us, position)); - while (raw_positions_at_time_copy.size() > 1) { - if (static_cast(raw_positions_at_time_copy.back().first) < - static_cast(time_us) - options_.filtering_time_window_us()) { - raw_positions_at_time_copy.pop_back(); - } else { - break; - } - } - - int filtered_position = Median(raw_positions_at_time_copy); - double delta_degs = - (filtered_position - current_position_px_) / pixels_per_degree_; - - // If the motion is smaller than the min_motion_to_reframe and camera is - // stationary, don't use the update. - if (IsMotionTooSmall(delta_degs) && !motion_state_) { - *state = false; - } else if (abs(delta_degs) < options_.reframe_window() && motion_state_) { - // If the motion is smaller than the reframe_window and camera is moving, - // don't use the update. - *state = false; - } else { - // Apply new position, plus the reframe window size. - *state = true; - } - - return absl::OkStatus(); -} -absl::Status KinematicPathSolver::AddObservation(int position, - const uint64 time_us) { - if (!initialized_) { - if (position < min_location_) { - current_position_px_ = min_location_; - } else if (position > max_location_) { - current_position_px_ = max_location_; - } else { - current_position_px_ = position; - } - target_position_px_ = position; - motion_state_ = false; - mean_delta_t_ = -1; - raw_positions_at_time_.push_front( - std::pair(time_us, position)); - current_time_ = time_us; - initialized_ = true; - current_velocity_deg_per_s_ = 0; - RET_CHECK_GT(pixels_per_degree_, 0) - << "pixels_per_degree must be larger than 0."; - RET_CHECK_GE(options_.update_rate_seconds(), 0) - << "update_rate_seconds must be greater than 0."; - RET_CHECK_GE(options_.filtering_time_window_us(), 0) - << "update_rate_seconds must be greater than 0."; - RET_CHECK_GE(options_.mean_period_update_rate(), 0) - << "mean_period_update_rate must be greater than 0."; - RET_CHECK(options_.has_min_motion_to_reframe() ^ - (options_.has_min_motion_to_reframe_upper() && - options_.has_min_motion_to_reframe_lower())) - << "Must set min_motion_to_reframe or min_motion_to_reframe_upper and " - "min_motion_to_reframe_lower."; - if (options_.has_min_motion_to_reframe()) { - RET_CHECK_GE(options_.min_motion_to_reframe(), options_.reframe_window()) - << "Reframe window cannot exceed min_motion_to_reframe."; - } else { - RET_CHECK_GE(options_.min_motion_to_reframe_upper(), - options_.reframe_window()) - << "Reframe window cannot exceed min_motion_to_reframe."; - RET_CHECK_GE(options_.min_motion_to_reframe_lower(), - options_.reframe_window()) - << "Reframe window cannot exceed min_motion_to_reframe."; - } - return absl::OkStatus(); - } - - RET_CHECK(current_time_ < time_us) - << "Observation added before a prior observations."; - - raw_positions_at_time_.push_front(std::pair(time_us, position)); - while (raw_positions_at_time_.size() > 1) { - if (static_cast(raw_positions_at_time_.back().first) < - static_cast(time_us) - options_.filtering_time_window_us()) { - raw_positions_at_time_.pop_back(); - } else { - break; - } - } - - int filtered_position = Median(raw_positions_at_time_); - double delta_degs = - (filtered_position - current_position_px_) / pixels_per_degree_; - - // If the motion is smaller than the min_motion_to_reframe and camera is - // stationary, don't use the update. - if (IsMotionTooSmall(delta_degs) && !motion_state_) { - delta_degs = 0; - motion_state_ = false; - } else if (abs(delta_degs) < options_.reframe_window() && motion_state_) { - // If the motion is smaller than the reframe_window and camera is moving, - // don't use the update. - delta_degs = 0; - motion_state_ = false; - } else if (delta_degs > 0) { - // Apply new position, less the reframe window size. - target_position_px_ = - filtered_position - pixels_per_degree_ * options_.reframe_window(); - delta_degs = - (target_position_px_ - current_position_px_) / pixels_per_degree_; - motion_state_ = true; - } else { - // Apply new position, plus the reframe window size. - target_position_px_ = - filtered_position + pixels_per_degree_ * options_.reframe_window(); - delta_degs = - (target_position_px_ - current_position_px_) / pixels_per_degree_; - motion_state_ = true; - } - - // Time and position updates. - double delta_t = (time_us - current_time_) / 1000000.0; - // Time since last state/prediction update, smoothed by - // mean_period_update_rate. - if (mean_delta_t_ < 0) { - mean_delta_t_ = delta_t; - } else { - mean_delta_t_ = mean_delta_t_ * (1 - options_.mean_period_update_rate()) + - delta_t * options_.mean_period_update_rate(); - } - - // Observed velocity and then weighted update of this velocity. - double observed_velocity = delta_degs / delta_t; - double update_rate = std::min(mean_delta_t_ / options_.update_rate_seconds(), - options_.max_update_rate()); - double updated_velocity = current_velocity_deg_per_s_ * (1 - update_rate) + - observed_velocity * update_rate; - // Limited current velocity. - current_velocity_deg_per_s_ = - updated_velocity > 0 ? fmin(updated_velocity, options_.max_velocity()) - : fmax(updated_velocity, -options_.max_velocity()); - - // Update prediction based on time input. - return UpdatePrediction(time_us); -} - -absl::Status KinematicPathSolver::UpdatePrediction(const int64 time_us) { - RET_CHECK(current_time_ < time_us) - << "Prediction time added before a prior observation or prediction."; - - // Position update limited by min/max. - double update_position_px = - current_position_px_ + - current_velocity_deg_per_s_ * mean_delta_t_ * pixels_per_degree_; - - if (update_position_px < min_location_) { - current_position_px_ = min_location_; - current_velocity_deg_per_s_ = 0; - motion_state_ = false; - } else if (update_position_px > max_location_) { - current_position_px_ = max_location_; - current_velocity_deg_per_s_ = 0; - motion_state_ = false; - } else { - current_position_px_ = update_position_px; - } - current_time_ = time_us; - - return absl::OkStatus(); -} - -absl::Status KinematicPathSolver::GetState(int* position) { - RET_CHECK(initialized_) << "GetState called before first observation added."; - *position = round(current_position_px_); - return absl::OkStatus(); -} - -absl::Status KinematicPathSolver::GetTargetPosition(int* target_position) { - RET_CHECK(initialized_) - << "GetTargetPosition called before first observation added."; - *target_position = round(target_position_px_); - return absl::OkStatus(); -} - -absl::Status KinematicPathSolver::UpdatePixelsPerDegree( - const float pixels_per_degree) { - RET_CHECK_GT(pixels_per_degree, 0) - << "pixels_per_degree must be larger than 0."; - pixels_per_degree_ = pixels_per_degree; - return absl::OkStatus(); -} - -absl::Status KinematicPathSolver::UpdateMinMaxLocation(const int min_location, - const int max_location) { - RET_CHECK(initialized_) - << "UpdateMinMaxLocation called before first observation added."; - double prior_distance = max_location_ - min_location_; - double updated_distance = max_location - min_location; - double scale_change = updated_distance / prior_distance; - current_position_px_ = current_position_px_ * scale_change; - target_position_px_ = target_position_px_ * scale_change; - max_location_ = max_location; - min_location_ = min_location; - auto original_positions_at_time = raw_positions_at_time_; - raw_positions_at_time_.clear(); - for (auto position_at_time : original_positions_at_time) { - position_at_time.second = position_at_time.second * scale_change; - raw_positions_at_time_.push_front(position_at_time); - } - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h deleted file mode 100644 index 4f4b896e2..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.pb.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// Kinematic path solver class is a stateful 1d position estimator based loosely -// on a differential kalman filter that is specifically designed to control a -// camera. It utilizes a Kalman filters predict/update interface for estimating -// the best camera focus position and updating that estimate when a measurement -// is available. Tuning controls include: update_rate: how much to update the -// existing state with a new state. max_velocity: max speed of the state per -// second. min_motion_to_reframe: only updating the state if a measurement -// exceeds this threshold. -class KinematicPathSolver { - public: - KinematicPathSolver(const KinematicOptions& options, const int min_location, - const int max_location, float pixels_per_degree) - : options_(options), - min_location_(min_location), - max_location_(max_location), - initialized_(false), - pixels_per_degree_(pixels_per_degree) {} - // Add an observation (detection) at a position and time. - absl::Status AddObservation(int position, const uint64 time_us); - // Get the predicted position at a time. - absl::Status UpdatePrediction(const int64 time_us); - // Get the state at a time. - absl::Status GetState(int* position); - // Update PixelPerDegree value. - absl::Status UpdatePixelsPerDegree(const float pixels_per_degree); - // Provide the current target position of the reframe action. - absl::Status GetTargetPosition(int* target_position); - // Change min/max location and update state based on new scaling. - absl::Status UpdateMinMaxLocation(const int min_location, - const int max_location); - // Check if motion is within the reframe window, return false if not. - bool IsMotionTooSmall(double delta_degs); - // Check if a position measurement will cause the camera to be in motion - // without updating the internal state. - absl::Status PredictMotionState(int position, const uint64 time_us, - bool* state); - // Clear any history buffer of positions that are used when - // filtering_time_window_us is set to a non-zero value. - void ClearHistory(); - - private: - // Tuning options. - KinematicOptions options_; - // Min and max value the state can be. - int min_location_; - int max_location_; - bool initialized_; - float pixels_per_degree_; - // Current state values. - double current_position_px_; - double current_velocity_deg_per_s_; - uint64 current_time_; - // History of observations (second) and their time (first). - std::deque> raw_positions_at_time_; - // Current target position. - double target_position_px_; - // Defines if the camera is moving to a target (true) or reached a target - // within a tolerance (false). - bool motion_state_; - // Average period of incoming frames. - double mean_delta_t_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto deleted file mode 100644 index 9f481db6d..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto +++ /dev/null @@ -1,33 +0,0 @@ -syntax = "proto2"; - -package mediapipe.autoflip; - -message KinematicOptions { - // Weighted update of new camera velocity (measurement) vs current state - // (prediction). - optional double update_rate = 1 [default = 0.5, deprecated = true]; - // Max velocity (degrees per second) that the camera can move. - optional double max_velocity = 2 [default = 18]; - // Min motion (in degrees) to react for both upper and lower directions. Must - // not be set if using min_motion_to_reframe_lower and - // min_motion_to_reframe_upper. - optional float min_motion_to_reframe = 3; - // Min motion (in degrees) for upper and lower direction to react. Both must - // be set and min_motion_to_reframe cannot be set if these are specified. - optional float min_motion_to_reframe_lower = 9; - optional float min_motion_to_reframe_upper = 10; - // When motion exceeds min_motion_to_reframe, move within this distance of the - // camera from the starting direction. Setting this value non-zero reduces - // total reframe distance on average. Value cannot exceed - // min_motion_to_reframe value. - optional float reframe_window = 4 [default = 0]; - // Calculation of internal velocity state is: - // min((delta_time_s / update_rate_seconds), max_update_rate) - // where delta_time_s is the time since the last frame. - optional double update_rate_seconds = 5 [default = 0.20]; - optional double max_update_rate = 6 [default = 0.8]; - // History time window of observations to be median filtered. - optional int64 filtering_time_window_us = 7 [default = 0]; - // Weighted update of average period, used for motion updates. - optional float mean_period_update_rate = 8 [default = 0.25]; -} diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc deleted file mode 100644 index 2a7665f66..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h" - -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -constexpr int64 kMicroSecInSec = 1000000; -constexpr float kWidthFieldOfView = 60; - -namespace mediapipe { -namespace autoflip { -namespace { - -TEST(KinematicPathSolverTest, FailZeroPixelsPerDegree) { - KinematicOptions options; - KinematicPathSolver solver(options, 0, 1000, 0); - EXPECT_FALSE(solver.AddObservation(500, kMicroSecInSec * 0).ok()); -} - -TEST(KinematicPathSolverTest, FailNotInitializedState) { - KinematicOptions options; - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - EXPECT_FALSE(solver.GetState(&state).ok()); -} - -TEST(KinematicPathSolverTest, FailNotInitializedPrediction) { - KinematicOptions options; - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int64 timestamp = 0; - EXPECT_FALSE(solver.UpdatePrediction(timestamp).ok()); -} - -TEST(KinematicPathSolverTest, PassNotEnoughMotionLargeImg) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(2.0); - options.set_update_rate(1); - options.set_max_velocity(1000); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move. - EXPECT_EQ(state, 500); -} - -TEST(KinematicPathSolverTest, PassNotEnoughMotionSmallImg) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(2.0); - options.set_update_rate(1); - options.set_max_velocity(500); - // Set degrees / pixel to 8.3 - KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0)); - // Move target by 10px / 8.3 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move. - EXPECT_EQ(state, 400); -} - -TEST(KinematicPathSolverTest, PassEnoughMotionFiltered) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1); - options.set_max_velocity(1000); - options.set_filtering_time_window_us(3000000); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2)); - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move. - EXPECT_EQ(state, 500); -} - -TEST(KinematicPathSolverTest, PassEnoughMotionNotFiltered) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1); - options.set_max_velocity(1000); - options.set_filtering_time_window_us(0); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2)); - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move. - EXPECT_EQ(state, 506); -} - -TEST(KinematicPathSolverTest, PassEnoughMotionLargeImg) { - KinematicOptions options; - // Set min motion to 1deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.0000001); - options.set_max_update_rate(1.0); - options.set_max_velocity(1000); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to move. - EXPECT_EQ(state, 520); -} - -TEST(KinematicPathSolverTest, PassEnoughMotionSmallImg) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.0000001); - options.set_max_update_rate(1.0); - options.set_max_velocity(18); - // Set degrees / pixel to 8.3 - KinematicPathSolver solver(options, 0, 500, 500.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(400, kMicroSecInSec * 0)); - // Move target by 10px / 8.3 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(410, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to move. - EXPECT_EQ(state, 410); -} - -TEST(KinematicPathSolverTest, FailReframeWindowSetting) { - KinematicOptions options; - // Set min motion to 1deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1); - options.set_max_velocity(1000); - // Set reframe window size to .75 for test. - options.set_reframe_window(1.1); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - ASSERT_FALSE(solver.AddObservation(500, kMicroSecInSec * 0).ok()); -} - -TEST(KinematicPathSolverTest, PassReframeWindow) { - KinematicOptions options; - // Set min motion to 1deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.0000001); - options.set_max_update_rate(1.0); - options.set_max_velocity(1000); - // Set reframe window size to .75 for test. - options.set_reframe_window(0.75); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to move 1.2-.75 deg, * 16.6 = 7.47px + 500 = - EXPECT_EQ(state, 508); -} - -TEST(KinematicPathSolverTest, PassReframeWindowLowerUpper) { - KinematicOptions options; - // Set min motion to 1deg - options.set_min_motion_to_reframe_upper(1.3); - options.set_min_motion_to_reframe_lower(1.0); - options.set_update_rate_seconds(.0000001); - options.set_max_update_rate(1.0); - options.set_max_velocity(1000); - // Set reframe window size to .75 for test. - options.set_reframe_window(0.75); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move - EXPECT_EQ(state, 500); - MP_ASSERT_OK(solver.AddObservation(480, kMicroSecInSec * 2)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to move - EXPECT_EQ(state, 493); -} - -TEST(KinematicPathSolverTest, PassCheckState) { - KinematicOptions options; - // Set min motion to 1deg - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.0000001); - options.set_max_update_rate(1.0); - options.set_max_velocity(1000); - // Set reframe window size to .75 for test. - options.set_reframe_window(0.75); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - bool motion_state; - MP_ASSERT_OK( - solver.PredictMotionState(520, kMicroSecInSec * 1, &motion_state)); - EXPECT_TRUE(motion_state); -} - -TEST(KinematicPathSolverTest, PassUpdateRate30FPS) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.25); - options.set_max_update_rate(0.8); - options.set_max_velocity(18); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1 / 30)); - MP_ASSERT_OK(solver.GetState(&state)); - // (0.033 / .25) * 20 = - EXPECT_EQ(state, 503); -} - -TEST(KinematicPathSolverTest, PassUpdateRate10FPS) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(.25); - options.set_max_update_rate(0.8); - options.set_max_velocity(18); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1 / 10)); - MP_ASSERT_OK(solver.GetState(&state)); - // (0.1 / .25) * 20 = - EXPECT_EQ(state, 508); -} - -TEST(KinematicPathSolverTest, PassUpdateRate) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(4); - options.set_max_update_rate(1.0); - options.set_max_velocity(18); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state, target_position; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - MP_ASSERT_OK(solver.GetTargetPosition(&target_position)); - EXPECT_EQ(target_position, 500); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetTargetPosition(&target_position)); - EXPECT_EQ(target_position, 520); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 505); -} - -TEST(KinematicPathSolverTest, PassUpdateRateResolutionChange) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate_seconds(4); - options.set_max_update_rate(1.0); - options.set_max_velocity(18); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state, target_position; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - MP_ASSERT_OK(solver.GetTargetPosition(&target_position)); - EXPECT_EQ(target_position, 500); - MP_ASSERT_OK(solver.UpdateMinMaxLocation(0, 500)); - MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView)); - MP_ASSERT_OK(solver.AddObservation(520 * 0.5, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetTargetPosition(&target_position)); - EXPECT_EQ(target_position, 520 * 0.5); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 253); -} - -TEST(KinematicPathSolverTest, PassMaxVelocity) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1.0); - options.set_max_velocity(6); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - MP_ASSERT_OK(solver.AddObservation(1000, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 600); -} - -TEST(KinematicPathSolverTest, PassDegPerPxChange) { - KinematicOptions options; - // Set min motion to 2deg - options.set_min_motion_to_reframe(2.0); - options.set_update_rate(1); - options.set_max_velocity(1000); - // Set degrees / pixel to 16.6 - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); - // Move target by 20px / 16.6 = 1.2deg - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 1)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to not move. - EXPECT_EQ(state, 500); - MP_ASSERT_OK(solver.UpdatePixelsPerDegree(500.0 / kWidthFieldOfView)); - MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2)); - MP_ASSERT_OK(solver.GetState(&state)); - // Expect cam to move. - EXPECT_EQ(state, 516); -} - -TEST(KinematicPathSolverTest, NoTimestampSmoothing) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1.0); - options.set_max_velocity(6); - options.set_mean_period_update_rate(1.0); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, 0)); - MP_ASSERT_OK(solver.AddObservation(1000, 1000000)); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 600); - MP_ASSERT_OK(solver.AddObservation(1000, 2200000)); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 720); -} - -TEST(KinematicPathSolverTest, TimestampSmoothing) { - KinematicOptions options; - options.set_min_motion_to_reframe(1.0); - options.set_update_rate(1.0); - options.set_max_velocity(6); - options.set_mean_period_update_rate(0.05); - KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); - int state; - MP_ASSERT_OK(solver.AddObservation(500, 0)); - MP_ASSERT_OK(solver.AddObservation(1000, 1000000)); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 600); - MP_ASSERT_OK(solver.AddObservation(1000, 2200000)); - MP_ASSERT_OK(solver.GetState(&state)); - EXPECT_EQ(state, 701); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/math_utils.h b/mediapipe/examples/desktop/autoflip/quality/math_utils.h deleted file mode 100644 index bac990165..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/math_utils.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_ - -class MathUtil { - public: - // Clamps value to the range [low, high]. Requires low <= high. Returns false - // if this check fails, otherwise returns true. Caller should first check the - // returned boolean. - template // T models LessThanComparable. - static bool Clamp(const T& low, const T& high, const T& value, T* result) { - // Prevents errors in ordering the arguments. - if (low > high) { - return false; - } - if (high < value) { - *result = high; - } else if (value < low) { - *result = low; - } else { - *result = value; - } - return true; - } -}; - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_MATH_UTILS_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc b/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc deleted file mode 100644 index 3d489c395..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.cc +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h" - -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace autoflip { - -PaddingEffectGenerator::PaddingEffectGenerator(const int input_width, - const int input_height, - const double target_aspect_ratio, - bool scale_to_multiple_of_two) { - target_aspect_ratio_ = target_aspect_ratio; - const double input_aspect_ratio = - static_cast(input_width) / static_cast(input_height); - input_width_ = input_width; - input_height_ = input_height; - is_vertical_padding_ = input_aspect_ratio > target_aspect_ratio; - output_width_ = is_vertical_padding_ - ? std::round(target_aspect_ratio * input_height) - : input_width; - output_height_ = is_vertical_padding_ - ? input_height - : std::round(input_width / target_aspect_ratio); - if (scale_to_multiple_of_two) { - output_width_ = output_width_ / 2 * 2; - output_height_ = output_height_ / 2 * 2; - } -} - -absl::Status PaddingEffectGenerator::Process( - const ImageFrame& input_frame, const float background_contrast, - const int blur_cv_size, const float overlay_opacity, - ImageFrame* output_frame, const cv::Scalar* background_color_in_rgb) { - RET_CHECK_EQ(input_frame.Width(), input_width_); - RET_CHECK_EQ(input_frame.Height(), input_height_); - RET_CHECK(output_frame); - - cv::Mat original_image = formats::MatView(&input_frame); - // This is the canvas that we are going to draw the padding effect on to. - cv::Mat canvas(output_height_, output_width_, original_image.type()); - - const int effective_input_width = - is_vertical_padding_ ? input_width_ : input_height_; - const int effective_input_height = - is_vertical_padding_ ? input_height_ : input_width_; - const int effective_output_width = - is_vertical_padding_ ? output_width_ : output_height_; - const int effective_output_height = - is_vertical_padding_ ? output_height_ : output_width_; - - if (!is_vertical_padding_) { - original_image = original_image.t(); - canvas = canvas.t(); - } - - const int foreground_height = - effective_input_height * effective_output_width / effective_input_width; - int x = -1, y = -1, width = -1, height = -1; - - // The following steps does the padding operation, with several steps. - // #1, we prepare the background. If a solid background color is given, we use - // it directly. Otherwise, we first crop a region of size "output_width_ * - // output_height_" off of the original frame to become the background of - // the final frame, and then we blur it and adjust contrast and opacity. - if (background_color_in_rgb != nullptr) { - canvas = *background_color_in_rgb; - } else { - // Copy the original image to the background. - x = 0.5 * (effective_input_width - effective_output_width); - y = 0; - width = effective_output_width; - height = effective_output_height; - cv::Rect crop_window_for_background(x, y, width, height); - original_image(crop_window_for_background).copyTo(canvas); - - // Blur. - const int cv_size = - blur_cv_size % 2 == 1 ? blur_cv_size : (blur_cv_size + 1); - const cv::Size kernel(cv_size, cv_size); - // TODO: the larger the kernel size, the slower the blurring - // operation is. Consider running multiple sequential blurs with smaller - // sizes to simulate the effect of using a large size. This might be able to - // speed up the process. - x = 0; - width = effective_output_width; - const cv::Rect canvas_rect(0, 0, canvas.cols, canvas.rows); - // Blur the top region (above foreground). - y = 0; - height = (effective_output_height - foreground_height) / 2 + cv_size; - const cv::Rect top_blur_region = - cv::Rect(x, y, width, height) & canvas_rect; - if (top_blur_region.area() > 0) { - cv::Mat top_blurred = canvas(top_blur_region); - cv::GaussianBlur(top_blurred, top_blurred, kernel, 0, 0); - } - // Blur the bottom region (below foreground). - y = height + foreground_height - cv_size; - height = effective_output_height - y; - const cv::Rect bottom_blur_region = - cv::Rect(x, y, width, height) & canvas_rect; - if (bottom_blur_region.area() > 0) { - cv::Mat bottom_blurred = canvas(bottom_blur_region); - cv::GaussianBlur(bottom_blurred, bottom_blurred, kernel, 0, 0); - } - - const float kEqualThreshold = 0.0001f; - // Background contrast adjustment. - if (std::abs(background_contrast - 1.0f) > kEqualThreshold) { - canvas *= background_contrast; - } - - // Alpha blend a translucent black layer. - if (std::abs(overlay_opacity - 0.0f) > kEqualThreshold) { - cv::Mat overlay = cv::Mat::zeros(canvas.size(), canvas.type()); - cv::addWeighted(overlay, overlay_opacity, canvas, 1 - overlay_opacity, 0, - canvas); - } - } - - // #2, we crop the entire region off of the original frame. This will become - // the foreground in the final frame. - x = 0; - y = 0; - width = effective_input_width; - height = effective_input_height; - - cv::Rect crop_window_for_foreground(x, y, width, height); - - // #3, we specify a region of size computed as below in the final frame to - // embed the foreground that we obtained in #2. The aspect ratio of - // this region should be the same as the foreground, but with a - // smaller size. Therefore, the height and width are derived using - // the ratio of the sizes. - // - embed size: output_width_ * height (to be computed) - // - foreground: input_width * input_height - // - // The location of this region is horizontally centralized in the - // frame, and saturated in horizontal dimension. - x = 0; - y = (effective_output_height - foreground_height) / 2; - width = effective_output_width; - height = foreground_height; - - cv::Rect region_to_embed_foreground(x, y, width, height); - cv::Mat dst = canvas(region_to_embed_foreground); - cv::resize(original_image(crop_window_for_foreground), dst, dst.size()); - - if (!is_vertical_padding_) { - canvas = canvas.t(); - } - - output_frame->CopyPixelData(input_frame.Format(), canvas.cols, canvas.rows, - canvas.data, - ImageFrame::kDefaultAlignmentBoundary); - return absl::OkStatus(); -} - -cv::Rect PaddingEffectGenerator::ComputeOutputLocation() { - const int effective_input_width = - is_vertical_padding_ ? input_width_ : input_height_; - const int effective_input_height = - is_vertical_padding_ ? input_height_ : input_width_; - const int effective_output_width = - is_vertical_padding_ ? output_width_ : output_height_; - const int effective_output_height = - is_vertical_padding_ ? output_height_ : output_width_; - - // Step 3 from "process" call above, compute foreground location. - const int foreground_height = - effective_input_height * effective_output_width / effective_input_width; - const int x = 0; - const int y = (effective_output_height - foreground_height) / 2; - const int width = effective_output_width; - const int height = foreground_height; - - cv::Rect region_to_embed_foreground(x, y, width, height); - - return region_to_embed_foreground; -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h b/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h deleted file mode 100644 index 2d33593b6..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_ - -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// Generates padding effects given input frames. Depending on where the padded -// contents are added, there are two cases: -// 1) Pad on the top and bottom of the input frame, aka vertical padding, i.e. -// input_aspect_ratio > target_aspect_ratio. In this case, output frames will -// have the same height as input frames, and the width will be adjusted to -// match the target aspect ratio. -// 2) Pad on the left and right of the input frame, aka horizontal padding, i.e. -// input_aspect_ratio < target_aspect_ratio. In this case, output frames will -// have the same width as original frames, and the height will be adjusted to -// match the target aspect ratio. -// If a background color is given, the background of the output frame will be -// filled with this solid color; otherwise, it is a blurred version of the input -// frame. -// -// Note: in both horizontal and vertical padding effects, the output frame size -// will be at most as large as the input frame size, with one dimension the -// same as the input (horizontal padding: width, vertical padding: height). If -// you intented to have the output frame be larger, you could add a -// ScaleImageCalculator as an upstream node before calling this calculator in -// your MediaPipe graph (not as a downstream node, because visual details may -// lose after appling the padding effect). -class PaddingEffectGenerator { - public: - // Always outputs width and height that are divisible by 2 if - // scale_to_multiple_of_two is set to true. - PaddingEffectGenerator(const int input_width, const int input_height, - const double target_aspect_ratio, - bool scale_to_multiple_of_two = false); - - // Apply the padding effect on the input frame. - // - blur_cv_size: The cv::Size() parameter used in creating blurry effects - // for padding backgrounds. - // - background_contrast: Contrast adjustment for padding background. This - // value should between 0 and 1, and the smaller the value, the darker the - // background. - // - overlay_opacity: In addition to adjusting the contrast, a translucent - // black layer will be alpha blended with the background. This value defines - // the opacity of the black layer. - // - background_color_in_rgb: If not null, uses this solid color as background - // instead of blurring the image, and does not adjust contrast or opacity. - absl::Status Process(const ImageFrame& input_frame, - const float background_contrast, const int blur_cv_size, - const float overlay_opacity, ImageFrame* output_frame, - const cv::Scalar* background_color_in_rgb = nullptr); - - // Compute the "render location" on the output frame where the "crop from" - // location is to be placed. For use with external rendering soutions. - cv::Rect ComputeOutputLocation(); - - private: - double target_aspect_ratio_; - int input_width_ = -1; - int input_height_ = -1; - int output_width_ = -1; - int output_height_ = -1; - bool is_vertical_padding_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PADDING_EFFECT_GENERATOR_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc b/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc deleted file mode 100644 index 787baa370..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/padding_effect_generator_test.cc +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/padding_effect_generator.h" - -#include "absl/flags/flag.h" -#include "absl/strings/str_cat.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_builder.h" -#include "mediapipe/framework/port/status_matchers.h" - -ABSL_FLAG(std::string, input_image, "", "The path to an input image."); -ABSL_FLAG(std::string, output_folder, "", - "The folder to output test result images."); - -namespace mediapipe { -namespace autoflip { -namespace { - -// An 320x180 RGB test image. -constexpr char kTestImage[] = - "mediapipe/examples/desktop/autoflip/quality/testdata/" - "google.jpg"; -constexpr char kResultImagePrefix[] = - "mediapipe/examples/desktop/autoflip/quality/testdata/" - "result_"; - -const cv::Scalar kRed = cv::Scalar(255, 0, 0); - -void TestWithAspectRatio(const double aspect_ratio, - const cv::Scalar* background_color_in_rgb = nullptr) { - std::string test_image; - const bool process_arbitrary_image = - !absl::GetFlag(FLAGS_input_image).empty(); - if (!process_arbitrary_image) { - std::string test_image_path = mediapipe::file::JoinPath("./", kTestImage); - MP_ASSERT_OK(mediapipe::file::GetContents(test_image_path, &test_image)); - } else { - MP_ASSERT_OK(mediapipe::file::GetContents(absl::GetFlag(FLAGS_input_image), - &test_image)); - } - - const std::vector contents_vector(test_image.begin(), test_image.end()); - cv::Mat decoded_mat = - cv::imdecode(contents_vector, -1 /* return the loaded image as-is */); - - ImageFormat::Format image_format = ImageFormat::UNKNOWN; - cv::Mat output_mat; - switch (decoded_mat.channels()) { - case 1: - image_format = ImageFormat::GRAY8; - output_mat = decoded_mat; - break; - case 3: - image_format = ImageFormat::SRGB; - cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB); - break; - case 4: - MP_ASSERT_OK(mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC) - << "4-channel image isn't supported yet"); - break; - default: - MP_ASSERT_OK(mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported number of channels: " - << decoded_mat.channels()); - } - std::unique_ptr test_frame = absl::make_unique( - image_format, decoded_mat.size().width, decoded_mat.size().height); - output_mat.copyTo(formats::MatView(test_frame.get())); - - PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(), - aspect_ratio); - ImageFrame result_frame; - MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame, - background_color_in_rgb)); - cv::Mat original_mat = formats::MatView(&result_frame); - cv::Mat input_mat; - switch (original_mat.channels()) { - case 1: - input_mat = original_mat; - break; - case 3: - // OpenCV assumes the image to be BGR order. To use imencode(), do color - // conversion first. - cv::cvtColor(original_mat, input_mat, cv::COLOR_RGB2BGR); - break; - case 4: - MP_ASSERT_OK(mediapipe::UnimplementedErrorBuilder(MEDIAPIPE_LOC) - << "4-channel image isn't supported yet"); - break; - default: - MP_ASSERT_OK(mediapipe::FailedPreconditionErrorBuilder(MEDIAPIPE_LOC) - << "Unsupported number of channels: " - << original_mat.channels()); - } - - std::vector parameters; - parameters.push_back(cv::IMWRITE_JPEG_QUALITY); - constexpr int kEncodingQuality = 75; - parameters.push_back(kEncodingQuality); - - std::vector encode_buffer; - // Note that imencode() will store the data in RGB order. - // Check its JpegEncoder::write() in "imgcodecs/src/grfmt_jpeg.cpp" for more - // info. - if (!cv::imencode(".jpg", input_mat, encode_buffer, parameters)) { - MP_ASSERT_OK(mediapipe::InternalErrorBuilder(MEDIAPIPE_LOC) - << "Fail to encode the image to be jpeg format."); - } - - std::string output_string(absl::string_view( - reinterpret_cast(&encode_buffer[0]), encode_buffer.size())); - - if (!process_arbitrary_image) { - std::string result_string_path = mediapipe::file::JoinPath( - "./", absl::StrCat(kResultImagePrefix, aspect_ratio, - background_color_in_rgb ? "_solid_background" : "", - ".jpg")); - std::string result_image; - MP_ASSERT_OK( - mediapipe::file::GetContents(result_string_path, &result_image)); - EXPECT_EQ(result_image, output_string); - } else { - std::string output_string_path = mediapipe::file::JoinPath( - absl::GetFlag(FLAGS_output_folder), - absl::StrCat("result_", aspect_ratio, - background_color_in_rgb ? "_solid_background" : "", - ".jpg")); - MP_ASSERT_OK( - mediapipe::file::SetContents(output_string_path, output_string)); - } -} - -TEST(PaddingEffectGeneratorTest, Success) { - TestWithAspectRatio(0.3); - TestWithAspectRatio(0.6); - TestWithAspectRatio(1.0); - TestWithAspectRatio(1.6); - TestWithAspectRatio(2.5); - TestWithAspectRatio(3.4); -} - -TEST(PaddingEffectGeneratorTest, SuccessWithBackgroundColor) { - TestWithAspectRatio(0.3, &kRed); - TestWithAspectRatio(0.6, &kRed); - TestWithAspectRatio(1.0, &kRed); - TestWithAspectRatio(1.6, &kRed); - TestWithAspectRatio(2.5, &kRed); - TestWithAspectRatio(3.4, &kRed); -} - -TEST(PaddingEffectGeneratorTest, ScaleToMultipleOfTwo) { - int input_width = 30; - int input_height = 30; - double target_aspect_ratio = 0.5; - int expect_width = 14; - int expect_height = input_height; - auto test_frame = absl::make_unique(/*format=*/ImageFormat::SRGB, - input_width, input_height); - - PaddingEffectGenerator generator(test_frame->Width(), test_frame->Height(), - target_aspect_ratio, - /*scale_to_multiple_of_two=*/true); - ImageFrame result_frame; - MP_ASSERT_OK(generator.Process(*test_frame, 0.3, 40, 0.0, &result_frame)); - EXPECT_EQ(result_frame.Width(), expect_width); - EXPECT_EQ(result_frame.Height(), expect_height); -} - -TEST(PaddingEffectGeneratorTest, ComputeOutputLocation) { - PaddingEffectGenerator generator(1920, 1080, 1.0); - - auto result_rect = generator.ComputeOutputLocation(); - EXPECT_EQ(result_rect.x, 0); - EXPECT_EQ(result_rect.y, 236); - EXPECT_EQ(result_rect.width, 1080); - EXPECT_EQ(result_rect.height, 607); -} -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc b/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc deleted file mode 100644 index fb8f44f11..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" - -#include - -#include -#include -#include - -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -void PiecewiseLinearFunction::AddPoint(double x, double y) { - if (!points_.empty()) { - CHECK_GE(x, points_.back().x) - << "Points must be provided in non-decreasing x order."; - } - points_.push_back(PiecewiseLinearFunction::Point(x, y)); -} - -std::vector::const_iterator -PiecewiseLinearFunction::GetIntervalIterator(double input) const { - PiecewiseLinearFunction::Point input_point(input, 0); - std::vector::const_iterator iter = - std::lower_bound(points_.begin(), points_.end(), input_point, - PointCompare()); - return iter; -} - -double PiecewiseLinearFunction::Interpolate( - const PiecewiseLinearFunction::Point& p1, - const PiecewiseLinearFunction::Point& p2, double input) const { - CHECK_LT(p1.x, input); - CHECK_GE(p2.x, input); - - return p2.y - (p2.x - input) / (p2.x - p1.x) * (p2.y - p1.y); -} - -double PiecewiseLinearFunction::Evaluate(double const input) const { - std::vector::const_iterator i = - GetIntervalIterator(input); - if (i == points_.begin()) { - return points_.front().y; - } - if (i == points_.end()) { - return points_.back().y; - } - - std::vector::const_iterator prev = i - 1; - return Interpolate(*prev, *i, input); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h b/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h deleted file mode 100644 index 67311a8c0..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_ - -#include - -namespace mediapipe { -namespace autoflip { - -// Implementation of piecewise linear functions. The function is specified as a -// series of points (x1,y1), (x2,y2),..., (xn,yn). It can be constructed -// programmatically by repeatedly calling the AddPoint(x, y) method. -class PiecewiseLinearFunction { - public: - PiecewiseLinearFunction() {} - - // Evaluate the function at the specified input. The output - // saturates at the values of the first and last interpolation - // points. - // f(x) = y1 for x <= x1 // Saturate at the lowest value - // f(x) = yn for x > xn // Saturate at the highest value - // f(x) = (x-xj)/(xk-xj)*(yk-yj) + yk for xj < x <= xk and k = j+1 - double Evaluate(double input) const; - - // Adds the given point to the function. Points must be added in - // non-decreasing x order. Because the points are given in sorted - // order, this function can be used to construct discontinuous - // functions. For example, if one defines - // f.AddPoint(-1.0, 0.0) - // f.AddPoint( 0.0, 0.0) - // f.AddPoint( 0.0, 1.0) - // f.AddPoint( 1.0, 1.0) - // the result function f is discontinuous at 0.0. By convention, - // this function will return f.Evaluate(0.0) = 0.0, and - // f.Evaluate(1e-12) = 1.0. This convention corresponds to the - // natural behavior of GetIntervalIterator(). - void AddPoint(double x, double y); - - private: - struct Point { - double x; - double y; - Point(double X, double Y) : x(X), y(Y) {} - }; - - // A functor for use with stl algorithms like sort() and lower_bound() that - // sorts by the point's x value. - class PointCompare { - public: - bool operator()(const Point& p1, const Point& p2) const { - return p1.x < p2.x; - } - }; - - // Returns the iterator, i, closest to points_.begin() such that - // input <= i->x or it returns points_.end() if input > all x values - // in points_. - std::vector::const_iterator GetIntervalIterator(double input) const; - - // Given two points p1 and p2 such that p1.x < input and p2.x >= input this - // returns the linear interpolation of the y value. - double Interpolate(const Point& p1, const Point& p2, double input) const; - - std::vector points_; -}; - -} // namespace autoflip -} // namespace mediapipe -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_PIECEWISE_LINEAR_FUNCTION_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc b/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc deleted file mode 100644 index 41978147c..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function_test.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" - -#include - -#include - -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" - -namespace { - -using mediapipe::autoflip::PiecewiseLinearFunction; - -// It should be OK to pass a spec that's out of order as it gets sorted. -TEST(PiecewiseLinearFunctionTest, ReordersSpec) { - PiecewiseLinearFunction f; - // This defines the line y = x between 0 and 5 - f.AddPoint(0, 0); - f.AddPoint(1, 1); - f.AddPoint(2, 2); - f.AddPoint(3, 3); - f.AddPoint(5, 5); - - // Should be 0 as -1 is less than the smallest x value in the spec so it - // should saturate. - ASSERT_EQ(0, f.Evaluate(-1)); - - // These shoud all be on the line y = x - ASSERT_EQ(0, f.Evaluate(0)); - ASSERT_EQ(0.5, f.Evaluate(0.5)); - ASSERT_EQ(4.5, f.Evaluate(4.5)); - ASSERT_EQ(5, f.Evaluate(5)); - - // Saturating on the high end. - ASSERT_EQ(5, f.Evaluate(6)); -} - -TEST(PiecewiseLinearFunctionTest, TestAddPoints) { - PiecewiseLinearFunction function; - function.AddPoint(0.0, 0.0); - function.AddPoint(1.0, 1.0); - EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0)); - EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0)); - EXPECT_DOUBLE_EQ(0.25, function.Evaluate(0.25)); -} - -TEST(PiecewiseLinearFunctionTest, AddPointsDiscontinuous) { - PiecewiseLinearFunction function; - function.AddPoint(-1.0, 0.0); - function.AddPoint(0.0, 0.0); - function.AddPoint(0.0, 1.0); - function.AddPoint(1.0, 1.0); - EXPECT_DOUBLE_EQ(0.0, function.Evaluate(-1.0)); - EXPECT_DOUBLE_EQ(0.0, function.Evaluate(0.0)); - EXPECT_DOUBLE_EQ(1.0, function.Evaluate(1e-12)); - EXPECT_DOUBLE_EQ(1.0, function.Evaluate(3.14)); -} - -} // namespace diff --git a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc b/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc deleted file mode 100644 index dd30566c2..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.cc +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h" - -#include "ceres/autodiff_cost_function.h" -#include "ceres/cost_function.h" -#include "ceres/loss_function.h" -#include "ceres/solver.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -using ceres::AutoDiffCostFunction; -using ceres::CauchyLoss; -using ceres::CostFunction; -using ceres::Problem; -using ceres::Solve; -using ceres::Solver; - -namespace { - -// A residual operator that computes the error using polynomial fitting. -struct PolynomialResidual { - PolynomialResidual(double in, double out) : in_(in), out_(out) {} - - template - bool operator()(const T* const a, const T* const b, const T* const c, - const T* const d, const T* const k, T* residual) const { - residual[0] = out_ - a[0] * in_ - b[0] * in_ * in_ - - c[0] * in_ * in_ * in_ - d[0] * in_ * in_ * in_ * in_ - k[0]; - return true; - } - - private: - const double in_; - const double out_; -}; - -// Computes the amount of delta position change along the fitted polynomial -// curve, translates the delta from being relative to the origin of the original -// dimension to being relative to the center of the original dimension, then -// regulates the delta to avoid moving camera off the frame boundaries. -float ComputeDelta(const float in, const int original_dimension, - const int output_dimension, const double a, const double b, - const double c, const double d, const double k) { - // The value `out` here represents a normalized distance between the center of - // the output window and the origin of the original window. - float out = - a * in + b * in * in + c * in * in * in + d * in * in * in * in + k; - // Translate `out` to a pixel distance between the center of the output window - // and the center of the original window. This value can be negative, 0, or - // positive. - float delta = (out - 0.5) * original_dimension; - - // Make sure delta doesn't move the camera off the frame boundary. - const float max_delta = (original_dimension - output_dimension) / 2.0f; - if (delta > max_delta) { - delta = max_delta; - } else if (delta < -max_delta) { - delta = -max_delta; - } - return delta; -} -} // namespace - -void PolynomialRegressionPathSolver::AddCostFunctionToProblem( - const double in, const double out, Problem* problem, double* a, double* b, - double* c, double* d, double* k) { - // Creating a cost function, with 1D residual and 5 1D parameter blocks. This - // is what the "1, 1, 1, 1, 1, 1" std::string below means. - CostFunction* cost_function = - new AutoDiffCostFunction( - new PolynomialResidual(in, out)); - VLOG(1) << "------- adding " << in << ": " << out; - problem->AddResidualBlock(cost_function, new CauchyLoss(0.5), a, b, c, d, k); -} - -absl::Status PolynomialRegressionPathSolver::ComputeCameraPath( - const std::vector& focus_point_frames, - const std::vector& prior_focus_point_frames, - const int original_width, const int original_height, const int output_width, - const int output_height, std::vector* all_transforms) { - RET_CHECK_GE(original_width, output_width); - RET_CHECK_GE(original_height, output_height); - const bool should_solve_x_problem = original_width != output_width; - const bool should_solve_y_problem = original_height != output_height; - RET_CHECK_GT(focus_point_frames.size() + prior_focus_point_frames.size(), 0); - Problem problem_x, problem_y; - for (int i = 0; i < prior_focus_point_frames.size(); ++i) { - const auto& spf = prior_focus_point_frames[i]; - for (const auto& sp : spf.point()) { - const double center_x = sp.norm_point_x(); - const double center_y = sp.norm_point_y(); - const auto t = i; - if (should_solve_x_problem) { - AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_, - &xd_, &xk_); - } - if (should_solve_y_problem) { - AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_, - &yd_, &yk_); - } - } - } - for (int i = 0; i < focus_point_frames.size(); ++i) { - const auto& spf = focus_point_frames[i]; - for (const auto& sp : spf.point()) { - const double center_x = sp.norm_point_x(); - const double center_y = sp.norm_point_y(); - const auto t = i + prior_focus_point_frames.size(); - if (should_solve_x_problem) { - AddCostFunctionToProblem(t, center_x, &problem_x, &xa_, &xb_, &xc_, - &xd_, &xk_); - } - if (should_solve_y_problem) { - AddCostFunctionToProblem(t, center_y, &problem_y, &ya_, &yb_, &yc_, - &yd_, &yk_); - } - } - } - - Solver::Options options; - options.linear_solver_type = ceres::DENSE_QR; - - Solver::Summary summary_x, summary_y; - Solve(options, &problem_x, &summary_x); - Solve(options, &problem_y, &summary_y); - all_transforms->clear(); - for (int i = 0; - i < focus_point_frames.size() + prior_focus_point_frames.size(); i++) { - // Code below assigns values into an affine model, defined as: - // [1 0 dx] - // [0 1 dy] - // When the camera moves along x axis, we assign delta to dx; otherwise we - // assign delta to dy. - cv::Mat transform = cv::Mat::eye(2, 3, CV_32FC1); - const float in = static_cast(i); - if (should_solve_x_problem) { - const float delta = ComputeDelta(in, original_width, output_width, xa_, - xb_, xc_, xd_, xk_); - transform.at(0, 2) = delta; - } - if (should_solve_y_problem) { - const float delta = ComputeDelta(in, original_height, output_height, ya_, - yb_, yc_, yd_, yk_); - transform.at(1, 2) = delta; - } - all_transforms->push_back(transform); - } - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h b/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h deleted file mode 100644 index a510169db..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_ - -#include "ceres/problem.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -class PolynomialRegressionPathSolver { - public: - PolynomialRegressionPathSolver() - : xa_(0.0), - xb_(0.0), - xc_(0.0), - xd_(0.0), - xk_(0.0), - ya_(0.0), - yb_(0.0), - yc_(0.0), - yd_(0.0), - yk_(0.0) {} - - // Given a series of focus points on frames, uses polynomial regression to - // compute a best guess of a 1D camera movement trajectory along x-axis and - // y-axis, such that focus points can be preserved as much as possible. The - // returned |all_transforms| hold the camera location at each timestamp - // corresponding to each input frame. - absl::Status ComputeCameraPath( - const std::vector& focus_point_frames, - const std::vector& prior_focus_point_frames, - const int original_width, const int original_height, - const int output_width, const int output_height, - std::vector* all_transforms); - - private: - // Adds a new cost function, constructed using |in| and |out|, into |problem|. - void AddCostFunctionToProblem(const double in, const double out, - ceres::Problem* problem, double* a, double* b, - double* c, double* d, double* k); - - // The current implementation fixes the polynomial order at 4, i.e. the - // equation to estimate is: out = a * in + b * in^2 + c * in^3 + d * in^4 + k. - // The two sets of parameters below are for estimating trajectories along - // x-axis and y-axis, respectively. - double xa_, xb_, xc_, xd_, xk_; - double ya_, yb_, yc_, yd_, yk_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_POLYNOMIAL_REGRESSION_PATH_SOLVER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc b/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc deleted file mode 100644 index c21245cde..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h" - -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -// A series of focus point locations from a real video. -constexpr int kNumObservations = 291; -constexpr double data[] = { - 1, 0.4072740648, 2, 0.406096287, 3, 0.4049185093, 4, 0.4037407361, - 5, 0.402562963, 6, 0.4013851806, 7, 0.4002074074, 8, 0.399029625, - 9, 0.3978518519, 10, 0.3966740741, 11, 0.3954963056, 12, 0.3943185231, - 13, 0.39314075, 14, 0.3919629676, 15, 0.3907851944, 16, 0.3896074167, - 17, 0.3884296435, 18, 0.3872518611, 19, 0.386074088, 20, 0.3848963194, - 21, 0.3837185417, 22, 0.3825407685, 23, 0.3813629861, 24, 0.3801852037, - 25, 0.3790074398, 26, 0.3778296574, 27, 0.376651875, 28, 0.3754741111, - 29, 0.3742963287, 30, 0.3731185509, 31, 0.3719407685, 32, 0.3707629861, - 33, 0.3695852222, 34, 0.3684074398, 35, 0.3672296759, 36, 0.3661564352, - 37, 0.3651877315, 38, 0.3643235509, 39, 0.3635639213, 40, 0.3629088241, - 41, 0.36235825, 42, 0.3619122222, 43, 0.3615707269, 44, 0.3613337639, - 45, 0.3612013519, 46, 0.3611734583, 47, 0.3612500926, 48, 0.3614312778, - 49, 0.361717, 50, 0.3621072546, 51, 0.362602037, 52, 0.3632013519, - 53, 0.3639052083, 54, 0.3647136111, 55, 0.3656265417, 56, 0.3666440046, - 57, 0.3677659907, 58, 0.3689925139, 59, 0.3703235926, 60, 0.3716546528, - 61, 0.3729857269, 62, 0.374316787, 63, 0.3756478611, 64, 0.3769789259, - 65, 0.37831, 66, 0.3796410648, 67, 0.3809721296, 68, 0.3823031944, - 69, 0.3836342685, 70, 0.384965338, 71, 0.3862963981, 72, 0.3876274676, - 73, 0.388958537, 74, 0.3902290324, 75, 0.391438963, 76, 0.3925883241, - 77, 0.3936771204, 78, 0.3947053426, 79, 0.3956729954, 80, 0.396580088, - 81, 0.3974265972, 82, 0.3982125509, 83, 0.3989379259, 84, 0.3996027407, - 85, 0.4002069815, 86, 0.400750662, 87, 0.4012337639, 88, 0.4016562963, - 89, 0.4020182731, 90, 0.4023196667, 91, 0.4025604954, 92, 0.4027407593, - 93, 0.4028604491, 94, 0.4029195787, 95, 0.4029181296, 96, 0.4028561204, - 97, 0.4027407593, 98, 0.4026254028, 99, 0.4025100417, 100, 0.4023946852, - 101, 0.4022793241, 102, 0.402163963, 103, 0.4020486065, 104, 0.4019332454, - 105, 0.4018178889, 106, 0.4017025278, 107, 0.4015871759, 108, 0.4014718102, - 109, 0.4013564491, 110, 0.4012410972, 111, 0.4011257315, 112, 0.4010103704, - 113, 0.4008950185, 114, 0.400779662, 115, 0.4008743935, 116, 0.4011792083, - 117, 0.4016941157, 118, 0.4024191111, 119, 0.4033541944, 120, 0.4044993704, - 121, 0.4058546343, 122, 0.4074199815, 123, 0.4091954259, 124, 0.4111809537, - 125, 0.4133765741, 126, 0.415782287, 127, 0.4183980787, 128, 0.4212239676, - 129, 0.4242599352, 130, 0.427506, 131, 0.4309621528, 132, 0.4346283935, - 133, 0.4385047222, 134, 0.4425911407, 135, 0.4468876481, 136, 0.4513942454, - 137, 0.4561109306, 138, 0.4608276204, 139, 0.4655443056, 140, 0.4702609861, - 141, 0.4749776736, 142, 0.4796943574, 143, 0.4844110435, 144, 0.4891277287, - 145, 0.4938444148, 146, 0.4985611, 147, 0.5032777852, 148, 0.5079944708, - 149, 0.512711156, 150, 0.5174278403, 151, 0.5221445259, 152, 0.526861212, - 153, 0.5315778981, 154, 0.5362945833, 155, 0.5410112662, 156, 0.5457279537, - 157, 0.5504446435, 158, 0.5551613241, 159, 0.5598780093, 160, 0.5643503102, - 161, 0.5685782454, 162, 0.572561787, 163, 0.5763009583, 164, 0.5797957546, - 165, 0.583046162, 166, 0.5860521944, 167, 0.5888138611, 168, 0.5913311296, - 169, 0.5936040278, 170, 0.5956325463, 171, 0.5974166806, 172, 0.5989564491, - 173, 0.6002518287, 174, 0.6013028241, 175, 0.6021094491, 176, 0.6026716944, - 177, 0.6029895556, 178, 0.6030630556, 179, 0.602892162, 180, 0.6024768889, - 181, 0.6018172361, 182, 0.6009132083, 183, 0.5997648009, 184, 0.5983720139, - 185, 0.5967348519, 186, 0.595057662, 187, 0.5933804769, 188, 0.5917032963, - 189, 0.5900261065, 190, 0.588348912, 191, 0.5866717315, 192, 0.5849945417, - 193, 0.5833173519, 194, 0.5816401713, 195, 0.5799629861, 196, 0.578285787, - 197, 0.5766086111, 198, 0.5749314213, 199, 0.5732542315, 200, 0.5715770509, - 201, 0.5698998611, 202, 0.5682226667, 203, 0.5665454861, 204, 0.5648682963, - 205, 0.5631911157, 206, 0.5615139213, 207, 0.559989287, 208, 0.5586172083, - 209, 0.5573976713, 210, 0.5563306944, 211, 0.5554162662, 212, 0.5546543889, - 213, 0.5540450648, 214, 0.5535882917, 215, 0.5532840694, 216, 0.5531324028, - 217, 0.5531332824, 218, 0.5532867176, 219, 0.5535927083, 220, 0.5540512454, - 221, 0.5546623333, 222, 0.5554259769, 223, 0.5563421667, 224, 0.5574109148, - 225, 0.5586322083, 226, 0.5600060556, 227, 0.5615324583, 228, 0.563211412, - 229, 0.565042912, 230, 0.5670269722, 231, 0.5691635833, 232, 0.5714527315, - 233, 0.5738944491, 234, 0.576488713, 235, 0.5792355231, 236, 0.581982338, - 237, 0.5847291528, 238, 0.5874759676, 239, 0.5902227824, 240, 0.5929695972, - 241, 0.5957164074, 242, 0.5984632269, 243, 0.601210037, 244, 0.6039568565, - 245, 0.6067036667, 246, 0.6094504815, 247, 0.6121973056, 248, 0.6149441111, - 249, 0.6176909352, 250, 0.62043775, 251, 0.6231845556, 252, 0.6258408148, - 253, 0.628406537, 254, 0.6308817269, 255, 0.6332663426, 256, 0.6355604167, - 257, 0.6377639352, 258, 0.6398769259, 259, 0.6418993519, 260, 0.6438312407, - 261, 0.6456725787, 262, 0.6474233704, 263, 0.6490836111, 264, 0.650653287, - 265, 0.6521324444, 266, 0.653521037, 267, 0.6548190926, 268, 0.656026588, - 269, 0.6571435417, 270, 0.6581699537, 271, 0.6591058102, 272, 0.6599511204, - 273, 0.6607058796, 274, 0.6613700926, 275, 0.6620343056, 276, 0.6626985185, - 277, 0.6633627454, 278, 0.6640269537, 279, 0.6646911759, 280, 0.6653553843, - 281, 0.6660195972, 282, 0.6666838056, 283, 0.6673480278, 284, 0.6680122361, - 285, 0.668676463, 286, 0.6693406713, 287, 0.6700048935, 288, 0.6706691019, - 289, 0.6713333333, 290, 0.671997537, 291, 0.6726617454, -}; - -constexpr double prediction[] = { - 18.885935, 16.560495, 14.316487, 12.15229, 10.066307, 8.056951, - 6.1226487, 4.2618394, 2.4729967, 0.7545829, -0.894928, -2.47702, - -3.9931893, -5.4449024, -6.833619, -8.160782, -9.427822, -10.63615, - -11.78717, -12.882269, -13.922811, -14.910168, -15.845669, -16.730648, - -17.566418, -18.354284, -19.095533, -19.791435, -20.443249, -21.052212, - -21.619564, -22.146511, -22.634256, -23.08399, -23.496883, -23.874086, - -24.216753, -24.526012, -24.80297, -25.048738, -25.264395, -25.451015, - -25.609661, -25.74137, -25.84718, -25.928093, -25.985123, -26.01925, - -26.031458, -26.022684, -25.993896, -25.946003, -25.879932, -25.796581, - -25.696838, -25.581581, -25.451654, -25.307919, -25.151188, -24.982292, - -24.802023, -24.611176, -24.410517, -24.200804, -23.98278, -23.757189, - -23.52473, -23.286121, -23.042034, -22.79315, -22.540123, -22.283602, - -22.024214, -21.762579, -21.499294, -21.234953, -20.970118, -20.70536, - -20.441223, -20.178228, -19.916899, -19.65773, -19.401217, -19.147831, - -18.898027, -18.65226, -18.410952, -18.174517, -17.943365, -17.717875, - -17.498428, -17.285383, -17.079079, -16.879856, -16.688019, -16.503883, - -16.327726, -16.15982, -16.000439, -15.849811, -15.7081785, -15.575754, - -15.452737, -15.339321, -15.235674, -15.141964, -15.058327, -14.9848995, - -14.9218025, -14.869129, -14.826971, -14.795404, -14.774489, -14.764267, - -14.764768, -14.776015, -14.798016, -14.830744, -14.874178, -14.9282875, - -14.993011, -15.068281, -15.15401, -15.250105, -15.356457, -15.472937, - -15.599405, -15.73571, -15.881681, -16.03713, -16.201872, -16.37569, - -16.558355, -16.749626, -16.94926, -17.156982, -17.372507, -17.595535, - -17.825771, -18.062872, -18.306505, -18.55632, -18.811947, -19.072998, - -19.339085, -19.609785, -19.884687, -20.16334, -20.4453, -20.730091, - -21.017235, -21.306234, -21.59658, -21.887743, -22.179192, -22.470362, - -22.760695, -23.049604, -23.336494, -23.620754, -23.901754, -24.17887, - -24.451435, -24.718779, -24.980236, -25.235092, -25.482649, -25.722181, - -25.952942, -26.174181, -26.38514, -26.585024, -26.77304, -26.948387, - -27.110231, -27.257734, -27.39005, -27.506304, -27.605618, -27.68709, - -27.749819, -27.792877, -27.81533, -27.816212, -27.794569, -27.749413, - -27.679752, -27.584576, -27.462858, -27.313555, -27.135622, -26.927996, - -26.689583, -26.419294, -26.11602, -25.778639, -25.40601, -24.996979, - -24.550379, -24.06503, -}; - -void GenerateDataPointsFromRealVideo( - const int focus_point_frames_length, - const int prior_focus_point_frames_length, - std::vector* focus_point_frames, - std::vector* prior_focus_point_frames) { - CHECK(focus_point_frames_length + prior_focus_point_frames_length <= - kNumObservations); - for (int i = 0; i < prior_focus_point_frames_length; i++) { - FocusPoint sp; - sp.set_norm_point_x(data[i]); - FocusPointFrame spf; - *spf.add_point() = sp; - prior_focus_point_frames->push_back(spf); - } - for (int i = 0; i < focus_point_frames_length; i++) { - FocusPoint sp; - sp.set_norm_point_x(data[i]); - FocusPointFrame spf; - *spf.add_point() = sp; - focus_point_frames->push_back(spf); - } -} - -TEST(PolynomialRegressionPathSolverTest, SuccessInTrackingCameraMode) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 100, - /* prior_focus_point_frames_length = */ 100, - &focus_point_frames, - &prior_focus_point_frames); - constexpr int kFrameWidth = 200; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - MP_ASSERT_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight, - kCropWidth, kCropHeight, &all_xforms)); - ASSERT_EQ(all_xforms.size(), 200); - for (int i = 0; i < all_xforms.size(); i++) { - cv::Mat mat = all_xforms[i]; - EXPECT_FLOAT_EQ(mat.at(0, 2), prediction[i]); - } -} - -TEST(PolynomialRegressionPathSolverTest, SuccessInStationaryCameraMode) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - - constexpr int kFocusPointFramesLength = 100; - constexpr float kNormStationaryFocusPointX = 0.34f; - - for (int i = 0; i < kFocusPointFramesLength; i++) { - FocusPoint sp; - // Add a fixed normalized focus point location. - sp.set_norm_point_x(kNormStationaryFocusPointX); - FocusPointFrame spf; - *spf.add_point() = sp; - focus_point_frames.push_back(spf); - } - constexpr int kFrameWidth = 300; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - MP_ASSERT_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight, - kCropWidth, kCropHeight, &all_xforms)); - ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength); - - constexpr int kExpectedShift = -48; - for (int i = 0; i < all_xforms.size(); i++) { - cv::Mat mat = all_xforms[i]; - EXPECT_FLOAT_EQ(mat.at(0, 2), kExpectedShift); - } -} - -// Test the case where focus points are so close to boundaries that the amount -// of shifts would have moved the camera to go outside frame boundaries. In this -// case, the solver should regulate the camera position shift to keep it stay -// inside the viewport. -TEST(PolynomialRegressionPathSolverTest, SuccessWhenFocusPointCloseToBoundary) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - - constexpr int kFocusPointFramesLength = 100; - constexpr float kNormStationaryFocusPointX = 0.99f; - - for (int i = 0; i < kFocusPointFramesLength; i++) { - FocusPoint sp; - // Add a fixed normalized focus point location. - sp.set_norm_point_x(kNormStationaryFocusPointX); - FocusPointFrame spf; - *spf.add_point() = sp; - focus_point_frames.push_back(spf); - } - constexpr int kFrameWidth = 500; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - MP_ASSERT_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight, - kCropWidth, kCropHeight, &all_xforms)); - ASSERT_EQ(all_xforms.size(), kFocusPointFramesLength); - - // Regulate max delta change = (500 - 100) / 2. - constexpr int kExpectedShift = 200; - for (int i = 0; i < all_xforms.size(); i++) { - cv::Mat mat = all_xforms[i]; - EXPECT_FLOAT_EQ(mat.at(0, 2), kExpectedShift); - } -} - -TEST(PolynomialRegressionPathSolverTest, FewFramesShouldWork) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1, - /* prior_focus_point_frames_length = */ 1, - &focus_point_frames, - &prior_focus_point_frames); - constexpr int kFrameWidth = 200; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - MP_ASSERT_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight, - kCropWidth, kCropHeight, &all_xforms)); - ASSERT_EQ(all_xforms.size(), 2); -} - -TEST(PolynomialRegressionPathSolverTest, OneCurrentFrameShouldWork) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 1, - /* prior_focus_point_frames_length = */ 0, - &focus_point_frames, - &prior_focus_point_frames); - constexpr int kFrameWidth = 200; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - MP_ASSERT_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, kFrameWidth, kFrameHeight, - kCropWidth, kCropHeight, &all_xforms)); - ASSERT_EQ(all_xforms.size(), 1); -} - -TEST(PolynomialRegressionPathSolverTest, ZeroFrameShouldFail) { - PolynomialRegressionPathSolver solver; - std::vector focus_point_frames; - std::vector prior_focus_point_frames; - std::vector all_xforms; - GenerateDataPointsFromRealVideo(/* focus_point_frames_length = */ 0, - /* prior_focus_point_frames_length = */ 0, - &focus_point_frames, - &prior_focus_point_frames); - constexpr int kFrameWidth = 200; - constexpr int kFrameHeight = 300; - constexpr int kCropWidth = 100; - constexpr int kCropHeight = 300; - ASSERT_FALSE(solver - .ComputeCameraPath(focus_point_frames, - prior_focus_point_frames, kFrameWidth, - kFrameHeight, kCropWidth, kCropHeight, - &all_xforms) - .ok()); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc deleted file mode 100644 index 0bfe72548..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.cc +++ /dev/null @@ -1,443 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h" - -#include - -#include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h" -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/timestamp.h" - -namespace mediapipe { -namespace autoflip { - -absl::Status SceneCameraMotionAnalyzer::AnalyzeSceneAndPopulateFocusPointFrames( - const KeyFrameCropOptions& key_frame_crop_options, - const std::vector& key_frame_crop_results, - const int scene_frame_width, const int scene_frame_height, - const std::vector& scene_frame_timestamps, - const bool has_solid_color_background, - SceneKeyFrameCropSummary* scene_summary, - std::vector* focus_point_frames, - SceneCameraMotion* scene_camera_motion) { - has_solid_color_background_ = has_solid_color_background; - total_scene_frames_ = scene_frame_timestamps.size(); - MP_RETURN_IF_ERROR(AggregateKeyFrameResults( - key_frame_crop_options, key_frame_crop_results, scene_frame_width, - scene_frame_height, scene_summary)); - - const int64 scene_span_ms = - scene_frame_timestamps.empty() - ? 0 - : scene_frame_timestamps.back() - scene_frame_timestamps.front(); - const double scene_span_sec = TimestampDiff(scene_span_ms).Seconds(); - SceneCameraMotion camera_motion; - MP_RETURN_IF_ERROR(DecideCameraMotionType( - key_frame_crop_options, scene_span_sec, scene_frame_timestamps.back(), - scene_summary, &camera_motion)); - if (scene_summary->has_salient_region()) { - last_scene_with_salient_region_ = camera_motion; - time_since_last_salient_region_us_ = scene_frame_timestamps.back(); - } - if (scene_camera_motion != nullptr) { - *scene_camera_motion = camera_motion; - } - - return PopulateFocusPointFrames(*scene_summary, camera_motion, - scene_frame_timestamps, focus_point_frames); -} - -absl::Status SceneCameraMotionAnalyzer::ToUseSteadyMotion( - const float look_at_center_x, const float look_at_center_y, - const int crop_window_width, const int crop_window_height, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const { - scene_summary->set_crop_window_width(crop_window_width); - scene_summary->set_crop_window_height(crop_window_height); - auto* steady_motion = scene_camera_motion->mutable_steady_motion(); - steady_motion->set_steady_look_at_center_x(look_at_center_x); - steady_motion->set_steady_look_at_center_y(look_at_center_y); - return absl::OkStatus(); -} - -absl::Status SceneCameraMotionAnalyzer::ToUseSweepingMotion( - const float start_x, const float start_y, const float end_x, - const float end_y, const int crop_window_width, - const int crop_window_height, const double time_duration_in_sec, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const { - auto* sweeping_motion = scene_camera_motion->mutable_sweeping_motion(); - sweeping_motion->set_sweep_start_center_x(start_x); - sweeping_motion->set_sweep_start_center_y(start_y); - sweeping_motion->set_sweep_end_center_x(end_x); - sweeping_motion->set_sweep_end_center_y(end_y); - scene_summary->set_crop_window_width(crop_window_width); - scene_summary->set_crop_window_height(crop_window_height); - const auto sweeping_log = absl::StrFormat( - "Success rate %.2f is low - Camera is sweeping from (%.1f, %.1f) to " - "(%.1f, %.1f) in %.2f seconds.", - scene_summary->frame_success_rate(), start_x, start_y, end_x, end_y, - time_duration_in_sec); - VLOG(1) << sweeping_log; - return absl::OkStatus(); -} - -absl::Status SceneCameraMotionAnalyzer::DecideCameraMotionType( - const KeyFrameCropOptions& key_frame_crop_options, - const double scene_span_sec, const int64 end_time_us, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const { - RET_CHECK_GE(scene_span_sec, 0.0) << "Scene time span is negative."; - RET_CHECK_NE(scene_summary, nullptr) << "Scene summary is null."; - RET_CHECK_NE(scene_camera_motion, nullptr) << "Scene camera motion is null."; - const float scene_frame_center_x = scene_summary->scene_frame_width() / 2.0f; - const float scene_frame_center_y = scene_summary->scene_frame_height() / 2.0f; - - // If no frame has any focus region, that is, the scene has no focus - // regions, then default to look at the center. - if (!scene_summary->has_salient_region()) { - VLOG(1) << "No focus regions - camera is set to be steady on center."; - float no_salient_position_x = scene_frame_center_x; - float no_salient_position_y = scene_frame_center_y; - if (end_time_us - time_since_last_salient_region_us_ < - options_.duration_before_centering_us() && - last_scene_with_salient_region_.has_steady_motion()) { - no_salient_position_x = last_scene_with_salient_region_.steady_motion() - .steady_look_at_center_x(); - no_salient_position_y = last_scene_with_salient_region_.steady_motion() - .steady_look_at_center_y(); - } - MP_RETURN_IF_ERROR(ToUseSteadyMotion( - no_salient_position_x, no_salient_position_y, - scene_summary->crop_window_width(), scene_summary->crop_window_height(), - scene_summary, scene_camera_motion)); - return absl::OkStatus(); - } - - // Sweep across the scene when 1) success rate is too low, AND 2) the current - // scene is long enough. - if (options_.allow_sweeping() && !has_solid_color_background_ && - scene_summary->frame_success_rate() < - options_.minimum_success_rate_for_sweeping() && - scene_span_sec >= options_.minimum_scene_span_sec_for_sweeping()) { - float start_x = -1.0, start_y = -1.0, end_x = -1.0, end_y = -1.0; - if (options_.sweep_entire_frame()) { - if (scene_summary->crop_window_width() > - key_frame_crop_options.target_width()) { // horizontal sweeping - start_x = 0.0f; - start_y = scene_frame_center_y; - end_x = scene_summary->scene_frame_width(); - end_y = scene_frame_center_y; - } else { // vertical sweeping - start_x = scene_frame_center_x; - start_y = 0.0f; - end_x = scene_frame_center_x; - end_y = scene_summary->scene_frame_height(); - } - } else { - start_x = scene_summary->key_frame_center_min_x(); - start_y = scene_summary->key_frame_center_min_y(); - end_x = scene_summary->key_frame_center_max_x(); - end_y = scene_summary->key_frame_center_max_y(); - } - MP_RETURN_IF_ERROR(ToUseSweepingMotion( - start_x, start_y, end_x, end_y, key_frame_crop_options.target_width(), - key_frame_crop_options.target_height(), scene_span_sec, scene_summary, - scene_camera_motion)); - return absl::OkStatus(); - } - - // If scene motion is small, then look at a steady point in the scene. - if ((scene_summary->horizontal_motion_amount() < - options_.motion_stabilization_threshold_percent() && - scene_summary->vertical_motion_amount() < - options_.motion_stabilization_threshold_percent()) || - total_scene_frames_ == 1) { - return DecideSteadyLookAtRegion(key_frame_crop_options, scene_summary, - scene_camera_motion); - } - - // Otherwise, tracks the focus regions. - scene_camera_motion->mutable_tracking_motion(); - return absl::OkStatus(); -} - -// If there is no required focus region, looks at the middle of the center -// range, and snaps to the scene center if close. Otherwise, look at the center -// of the union of the required focus regions, and ensures the crop region -// covers this union. -absl::Status SceneCameraMotionAnalyzer::DecideSteadyLookAtRegion( - const KeyFrameCropOptions& key_frame_crop_options, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const { - const float scene_frame_width = scene_summary->scene_frame_width(); - const float scene_frame_height = scene_summary->scene_frame_height(); - const int target_width = key_frame_crop_options.target_width(); - const int target_height = key_frame_crop_options.target_height(); - float center_x = -1, center_y = -1; - float crop_width = -1, crop_height = -1; - - if (scene_summary->has_required_salient_region()) { - // Set look-at position to be the center of the union of required focus - // regions and the crop window size to be the maximum of this union size - // and the target size. - const auto& required_region_union = - scene_summary->key_frame_required_crop_region_union(); - center_x = required_region_union.x() + required_region_union.width() / 2.0f; - center_y = - required_region_union.y() + required_region_union.height() / 2.0f; - crop_width = std::max(target_width, required_region_union.width()); - crop_height = std::max(target_height, required_region_union.height()); - } else { - // Set look-at position to be the middle of the center range, and the crop - // window size to be the target size. - center_x = (scene_summary->key_frame_center_min_x() + - scene_summary->key_frame_center_max_x()) / - 2.0f; - center_y = (scene_summary->key_frame_center_min_y() + - scene_summary->key_frame_center_max_y()) / - 2.0f; - crop_width = target_width; - crop_height = target_height; - - // Optionally snap the look-at position to the scene frame center. - const float center_x_distance = - std::fabs(center_x - scene_frame_width / 2.0f); - const float center_y_distance = - std::fabs(center_y - scene_frame_height / 2.0f); - if (center_x_distance / scene_frame_width < - options_.snap_center_max_distance_percent()) { - center_x = scene_frame_width / 2.0f; - } - if (center_y_distance / scene_frame_height < - options_.snap_center_max_distance_percent()) { - center_y = scene_frame_height / 2.0f; - } - } - - // Clamp the region to be inside the frame. - // TODO: this may not be necessary. - float clamped_center_x, clamped_center_y; - RET_CHECK(MathUtil::Clamp(crop_width / 2.0f, - scene_frame_width - crop_width / 2.0f, center_x, - &clamped_center_x)); - center_x = clamped_center_x; - RET_CHECK(MathUtil::Clamp(crop_height / 2.0f, - scene_frame_height - crop_height / 2.0f, center_y, - &clamped_center_y)); - center_y = clamped_center_y; - - VLOG(1) << "Motion is small - camera is set to be steady at " << center_x - << ", " << center_y; - MP_RETURN_IF_ERROR(ToUseSteadyMotion(center_x, center_y, crop_width, - crop_height, scene_summary, - scene_camera_motion)); - return absl::OkStatus(); -} - -absl::Status SceneCameraMotionAnalyzer::AddFocusPointsFromCenterTypeAndWeight( - const float center_x, const float center_y, const int frame_width, - const int frame_height, const FocusPointFrameType type, const float weight, - const float bound, FocusPointFrame* focus_point_frame) const { - RET_CHECK_NE(focus_point_frame, nullptr) << "Focus point frame is null."; - const float norm_x = center_x / frame_width; - const float norm_y = center_y / frame_height; - const std::vector extremal_values = {0, 1}; - if (type == TOPMOST_AND_BOTTOMMOST) { - for (const float extremal_value : extremal_values) { - auto* focus_point = focus_point_frame->add_point(); - focus_point->set_norm_point_x(norm_x); - focus_point->set_norm_point_y(extremal_value); - focus_point->set_weight(weight); - focus_point->set_left(bound); - focus_point->set_right(bound); - } - } else if (type == LEFTMOST_AND_RIGHTMOST) { - for (const float extremal_value : extremal_values) { - auto* focus_point = focus_point_frame->add_point(); - focus_point->set_norm_point_x(extremal_value); - focus_point->set_norm_point_y(norm_y); - focus_point->set_weight(weight); - focus_point->set_top(bound); - focus_point->set_bottom(bound); - } - } else if (type == CENTER) { - auto* focus_point = focus_point_frame->add_point(); - focus_point->set_norm_point_x(norm_x); - focus_point->set_norm_point_y(norm_y); - focus_point->set_weight(weight); - focus_point->set_left(bound); - focus_point->set_right(bound); - focus_point->set_top(bound); - focus_point->set_bottom(bound); - } else { - RET_CHECK_FAIL() << absl::StrCat("Invalid FocusPointFrameType ", type); - } - return absl::OkStatus(); -} - -absl::Status SceneCameraMotionAnalyzer::PopulateFocusPointFrames( - const SceneKeyFrameCropSummary& scene_summary, - const SceneCameraMotion& scene_camera_motion, - const std::vector& scene_frame_timestamps, - std::vector* focus_point_frames) const { - RET_CHECK_NE(focus_point_frames, nullptr) - << "Output vector of FocusPointFrame is null."; - - const int num_scene_frames = scene_frame_timestamps.size(); - RET_CHECK_GT(num_scene_frames, 0) << "No scene frames."; - RET_CHECK_EQ(scene_summary.num_key_frames(), - scene_summary.key_frame_compact_infos_size()) - << "Key frame compact infos has wrong size:" - << " num_key_frames = " << scene_summary.num_key_frames() - << " key_frame_compact_infos size = " - << scene_summary.key_frame_compact_infos_size(); - const int scene_frame_width = scene_summary.scene_frame_width(); - const int scene_frame_height = scene_summary.scene_frame_height(); - RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width."; - RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height."; - - FocusPointFrameType focus_point_frame_type = - (scene_summary.crop_window_height() == scene_frame_height) - ? TOPMOST_AND_BOTTOMMOST - : (scene_summary.crop_window_width() == scene_frame_width - ? LEFTMOST_AND_RIGHTMOST - : CENTER); - focus_point_frames->reserve(num_scene_frames); - - if (scene_camera_motion.has_steady_motion()) { - // Camera focuses on a steady point of the scene. - const float center_x = - scene_camera_motion.steady_motion().steady_look_at_center_x(); - const float center_y = - scene_camera_motion.steady_motion().steady_look_at_center_y(); - for (int i = 0; i < num_scene_frames; ++i) { - FocusPointFrame focus_point_frame; - MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight( - center_x, center_y, scene_frame_width, scene_frame_height, - focus_point_frame_type, options_.maximum_salient_point_weight(), - options_.salient_point_bound(), &focus_point_frame)); - focus_point_frames->push_back(focus_point_frame); - } - return absl::OkStatus(); - } else if (scene_camera_motion.has_sweeping_motion()) { - // Camera sweeps across the frame. - const auto& sweeping_motion = scene_camera_motion.sweeping_motion(); - const float start_x = sweeping_motion.sweep_start_center_x(); - const float start_y = sweeping_motion.sweep_start_center_y(); - const float end_x = sweeping_motion.sweep_end_center_x(); - const float end_y = sweeping_motion.sweep_end_center_y(); - for (int i = 0; i < num_scene_frames; ++i) { - const float fraction = - num_scene_frames > 1 ? static_cast(i) / (num_scene_frames - 1) - : 0; - const float position_x = start_x * (1.0f - fraction) + end_x * fraction; - const float position_y = start_y * (1.0f - fraction) + end_y * fraction; - FocusPointFrame focus_point_frame; - MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight( - position_x, position_y, scene_frame_width, scene_frame_height, - focus_point_frame_type, options_.maximum_salient_point_weight(), - options_.salient_point_bound(), &focus_point_frame)); - focus_point_frames->push_back(focus_point_frame); - } - return absl::OkStatus(); - } else if (scene_camera_motion.has_tracking_motion()) { - // Camera tracks crop regions. - RET_CHECK_GT(scene_summary.num_key_frames(), 0) << "No key frames."; - return PopulateFocusPointFramesForTracking( - scene_summary, focus_point_frame_type, scene_frame_timestamps, - focus_point_frames); - } else { - return absl::Status(StatusCode::kInvalidArgument, "Unknown motion type."); - } -} - -// Linearly interpolates between key frames based on the timestamps using -// piecewise-linear functions for the crop region centers and scores. Adds one -// focus point at the center of the interpolated crop region for each frame. -// The weight for the focus point is proportional to the interpolated score -// and scaled so that the maximum weight is equal to -// maximum_focus_point_weight in the SceneCameraMotionAnalyzerOptions. -absl::Status SceneCameraMotionAnalyzer::PopulateFocusPointFramesForTracking( - const SceneKeyFrameCropSummary& scene_summary, - const FocusPointFrameType focus_point_frame_type, - const std::vector& scene_frame_timestamps, - std::vector* focus_point_frames) const { - RET_CHECK_GE(scene_summary.key_frame_max_score(), 0.0) - << "Maximum score is negative."; - - const int num_key_frames = scene_summary.num_key_frames(); - const auto& key_frame_compact_infos = scene_summary.key_frame_compact_infos(); - const int num_scene_frames = scene_frame_timestamps.size(); - const int scene_frame_width = scene_summary.scene_frame_width(); - const int scene_frame_height = scene_summary.scene_frame_height(); - - PiecewiseLinearFunction center_x_function, center_y_function, score_function; - const int64 timestamp_offset = key_frame_compact_infos[0].timestamp_ms(); - for (int i = 0; i < num_key_frames; ++i) { - const float center_x = key_frame_compact_infos[i].center_x(); - const float center_y = key_frame_compact_infos[i].center_y(); - const float score = key_frame_compact_infos[i].score(); - // Skips empty key frames. - if (center_x < 0 || center_y < 0 || score < 0) { - continue; - } - const double relative_timestamp = - key_frame_compact_infos[i].timestamp_ms() - timestamp_offset; - center_x_function.AddPoint(relative_timestamp, center_x); - center_y_function.AddPoint(relative_timestamp, center_y); - score_function.AddPoint(relative_timestamp, score); - } - - double max_score = 0.0; - const double min_score = 1e-4; // prevent constraints with 0 weight - for (int i = 0; i < num_scene_frames; ++i) { - const double relative_timestamp = - static_cast(scene_frame_timestamps[i] - timestamp_offset); - const double center_x = center_x_function.Evaluate(relative_timestamp); - const double center_y = center_y_function.Evaluate(relative_timestamp); - const double score = - std::max(min_score, score_function.Evaluate(relative_timestamp)); - max_score = std::max(max_score, score); - FocusPointFrame focus_point_frame; - MP_RETURN_IF_ERROR(AddFocusPointsFromCenterTypeAndWeight( - center_x, center_y, scene_frame_width, scene_frame_height, - focus_point_frame_type, score, options_.salient_point_bound(), - &focus_point_frame)); - focus_point_frames->push_back(focus_point_frame); - } - - // Scales weights so that maximum weight = maximum_salient_point_weight. - // TODO: run some experiments to find out if this is necessary. - max_score = std::max(max_score, min_score); - const double scale = options_.maximum_salient_point_weight() / max_score; - for (int i = 0; i < focus_point_frames->size(); ++i) { - for (int j = 0; j < (*focus_point_frames)[i].point_size(); ++j) { - auto* focus_point = (*focus_point_frames)[i].mutable_point(j); - focus_point->set_weight(scale * focus_point->weight()); - } - } - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h deleted file mode 100644 index 8688e16ed..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// This class does the following in order: -// - Aggregates a key frame results to get a SceneKeyFrameCropSummary, -// - Determines the SceneCameraMotion for the scene, and then -// - Populates FocusPointFrames to be used as input for the retargeter. -// -// Upstream inputs: -// - std::vector key_frame_crop_infos. -// - KeyFrameCropOptions key_frame_crop_options. -// - std::vector key_frame_crop_results. -// - int scene_frame_width, scene_frame_height. -// - std::vector scene_frame_timestamps. -// -// Example usage: -// SceneCameraMotionAnalyzerOptions options; -// SceneCameraMotionAnalyzer analyzer(options); -// SceneKeyFrameCropSummary scene_summary; -// std::vector focus_point_frames; -// CHECK_OK(analyzer.AnalyzeScenePopulateFocusPointFrames( -// key_frame_crop_infos, key_frame_crop_options, key_frame_crop_results, -// scene_frame_width, scene_frame_height, scene_frame_timestamps, -// &scene_summary, &focus_point_frames)); -class SceneCameraMotionAnalyzer { - public: - SceneCameraMotionAnalyzer() = delete; - - explicit SceneCameraMotionAnalyzer(const SceneCameraMotionAnalyzerOptions& - scene_camera_motion_analyzer_options) - : options_(scene_camera_motion_analyzer_options), - time_since_last_salient_region_us_(0), - has_solid_color_background_(false) {} - - ~SceneCameraMotionAnalyzer() {} - - // Aggregates information from KeyFrameInfos and KeyFrameCropResults into - // SceneKeyFrameCropSummary, and populates FocusPointFrames given scene - // frame timestamps. Optionally returns SceneCameraMotion. - absl::Status AnalyzeSceneAndPopulateFocusPointFrames( - const KeyFrameCropOptions& key_frame_crop_options, - const std::vector& key_frame_crop_results, - const int scene_frame_width, const int scene_frame_height, - const std::vector& scene_frame_timestamps, - const bool has_solid_color_background, - SceneKeyFrameCropSummary* scene_summary, - std::vector* focus_point_frames, - SceneCameraMotion* scene_camera_motion = nullptr); - - protected: - // Decides SceneCameraMotion based on SceneKeyFrameCropSummary. Updates the - // crop window in SceneKeyFrameCropSummary in the case of steady motion. - absl::Status DecideCameraMotionType( - const KeyFrameCropOptions& key_frame_crop_options, - const double scene_span_sec, const int64 end_time_us, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const; - - // Populates the FocusPointFrames for each scene frame based on - // SceneKeyFrameCropSummary, SceneCameraMotion, and scene frame timestamps. - absl::Status PopulateFocusPointFrames( - const SceneKeyFrameCropSummary& scene_summary, - const SceneCameraMotion& scene_camera_motion, - const std::vector& scene_frame_timestamps, - std::vector* focus_point_frames) const; - - private: - // Decides the look-at region when camera is steady. - absl::Status DecideSteadyLookAtRegion( - const KeyFrameCropOptions& key_frame_crop_options, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const; - - // Types of FocusPointFrames: number and placement of FocusPoint's vary. - enum FocusPointFrameType { - TOPMOST_AND_BOTTOMMOST = 1, // (center_x, 0) and (center_x, frame_height) - LEFTMOST_AND_RIGHTMOST = 2, // (0, center_y) and (frame_width, center_y) - CENTER = 3, // (center_x, center_y) - }; - - // Adds FocusPoint(s) to given FocusPointFrame given center location, - // frame size, FocusPointFrameType, weight, and bound. - absl::Status AddFocusPointsFromCenterTypeAndWeight( - const float center_x, const float center_y, const int frame_width, - const int frame_height, const FocusPointFrameType type, - const float weight, const float bound, - FocusPointFrame* focus_point_frame) const; - - // Populates the FocusPointFrames for each scene frame based on - // SceneKeyFrameCropSummary and scene frame timestamps in the case where - // camera is tracking the crop regions. - absl::Status PopulateFocusPointFramesForTracking( - const SceneKeyFrameCropSummary& scene_summary, - const FocusPointFrameType focus_point_frame_type, - const std::vector& scene_frame_timestamps, - std::vector* focus_point_frames) const; - - // Decide to use steady motion. - absl::Status ToUseSteadyMotion(const float look_at_center_x, - const float look_at_center_y, - const int crop_window_width, - const int crop_window_height, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const; - - // Decide to use sweeping motion. - absl::Status ToUseSweepingMotion( - const float start_x, const float start_y, const float end_x, - const float end_y, const int crop_window_width, - const int crop_window_height, const double time_duration_in_sec, - SceneKeyFrameCropSummary* scene_summary, - SceneCameraMotion* scene_camera_motion) const; - - // Scene camera motion analyzer options. - SceneCameraMotionAnalyzerOptions options_; - - // Last position - SceneCameraMotion last_scene_with_salient_region_; - int64 time_since_last_salient_region_us_; - - // Scene has solid color background. - bool has_solid_color_background_; - - // Total number of frames for this scene. - int total_scene_frames_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CAMERA_MOTION_ANALYZER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc deleted file mode 100644 index 703d17534..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc +++ /dev/null @@ -1,805 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h" - -#include -#include -#include -#include - -#include "absl/flags/flag.h" -#include "absl/strings/str_split.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" -#include "mediapipe/framework/deps/file_path.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { - -using ::testing::HasSubstr; - -const int kNumKeyFrames = 5; -const int kNumSceneFrames = 30; - -const int64 kKeyFrameTimestampDiff = 1e6 / kNumKeyFrames; -const int64 kSceneFrameTimestampDiff = 1e6 / kNumSceneFrames; -// Default time span of a scene in seconds. -const double kSceneTimeSpanSec = 1.0; - -const int kSceneFrameWidth = 100; -const int kSceneFrameHeight = 100; - -const int kTargetWidth = 50; -const int kTargetHeight = 50; - -constexpr char kCameraTrackingSceneFrameResultsFile[] = - "mediapipe/examples/desktop/autoflip/quality/testdata/" - "camera_motion_tracking_scene_frame_results.csv"; - -// Makes a rectangle given the corner (x, y) and the size (width, height). -Rect MakeRect(const int x, const int y, const int width, const int height) { - Rect rect; - rect.set_x(x); - rect.set_y(y); - rect.set_width(width); - rect.set_height(height); - return rect; -} - -// Returns default values for scene frame timestamps. Populates timestamps using -// the default spacing kSceneFrameTimestampDiff starting from 0. -std::vector GetDefaultSceneFrameTimestamps() { - std::vector scene_frame_timestamps(kNumSceneFrames); - for (int i = 0; i < kNumSceneFrames; ++i) { - scene_frame_timestamps[i] = kSceneFrameTimestampDiff * i; - } - return scene_frame_timestamps; -} - -// Returns default settings for KeyFrameCropOptions. Populates target size to be -// the default target size. -KeyFrameCropOptions GetDefaultKeyFrameCropOptions() { - KeyFrameCropOptions key_frame_crop_options; - key_frame_crop_options.set_target_width(kTargetWidth); - key_frame_crop_options.set_target_height(kTargetHeight); - return key_frame_crop_options; -} - -// Returns default values for KeyFrameCropResults. Sets each frame to have -// covered all the required regions and non-required regions, and have required -// crop region (10, 10+20) x (10, 10+20), (full) crop region (0, 50) x (0, 50), -// and region score 1.0. -std::vector GetDefaultKeyFrameCropResults() { - std::vector key_frame_crop_results(kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - key_frame_crop_results[i].set_are_required_regions_covered_in_target_size( - true); - key_frame_crop_results[i].set_fraction_non_required_covered(1.0); - key_frame_crop_results[i].set_region_is_empty(false); - key_frame_crop_results[i].set_required_region_is_empty(false); - *(key_frame_crop_results[i].mutable_region()) = MakeRect(0, 0, 50, 50); - *(key_frame_crop_results[i].mutable_required_region()) = - MakeRect(10, 10, 20, 20); - key_frame_crop_results[i].set_region_score(1.0); - key_frame_crop_results[i].set_timestamp_ms(kKeyFrameTimestampDiff * i); - } - return key_frame_crop_results; -} - -// Returns default settings for SceneKeyFrameCropSummary. Sets scene frame size -// to be the default size. Sets each key frame compact info in accordance to the -// default timestamps (using the default spacing kKeyFrameTimestampDiff starting -// from 0), default crop regions (centered at (25, 25)), and default scores -// (1.0). Sets center range to be [25, 25] and [25, 25]. Sets score range to be -// [1.0, 1.0]. Sets crop window size to be (25, 25). Sets has focus region to -// be true. Sets frame success rate to be 1.0. Sets horizontal and vertical -// motion amount to be 0.0. -SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() { - SceneKeyFrameCropSummary scene_summary; - scene_summary.set_scene_frame_width(kSceneFrameWidth); - scene_summary.set_scene_frame_height(kSceneFrameHeight); - scene_summary.set_num_key_frames(kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - auto* compact_info = scene_summary.add_key_frame_compact_infos(); - compact_info->set_timestamp_ms(kKeyFrameTimestampDiff * i); - compact_info->set_center_x(25); - compact_info->set_center_y(25); - compact_info->set_score(1.0); - } - scene_summary.set_key_frame_center_min_x(25); - scene_summary.set_key_frame_center_max_x(25); - scene_summary.set_key_frame_center_min_y(25); - scene_summary.set_key_frame_center_max_y(25); - scene_summary.set_key_frame_min_score(1.0); - scene_summary.set_key_frame_max_score(1.0); - scene_summary.set_crop_window_width(25); - scene_summary.set_crop_window_height(25); - scene_summary.set_has_salient_region(true); - scene_summary.set_frame_success_rate(1.0); - scene_summary.set_horizontal_motion_amount(0.0); - scene_summary.set_vertical_motion_amount(0.0); - return scene_summary; -} - -// Returns a SceneKeyFrameCropSummary with small motion. Sets crop window size -// to default target size. Sets horizontal motion to half the threshold in the -// options and vertical motion to 0. Sets center x range to [45, 55]. -SceneKeyFrameCropSummary GetSceneKeyFrameCropSummaryWithSmallMotion( - const SceneCameraMotionAnalyzerOptions& options) { - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_crop_window_width(kTargetWidth); - scene_summary.set_crop_window_height(kTargetHeight); - scene_summary.set_horizontal_motion_amount( - options.motion_stabilization_threshold_percent() / 2.0); - scene_summary.set_vertical_motion_amount(0.0); - scene_summary.set_key_frame_center_min_x(45); - scene_summary.set_key_frame_center_max_x(55); - return scene_summary; -} - -// Testable class that allows public access to protected methods in the class. -class TestableSceneCameraMotionAnalyzer : public SceneCameraMotionAnalyzer { - public: - explicit TestableSceneCameraMotionAnalyzer( - const SceneCameraMotionAnalyzerOptions& - scene_camera_motion_analyzer_options) - : SceneCameraMotionAnalyzer(scene_camera_motion_analyzer_options) {} - ~TestableSceneCameraMotionAnalyzer() {} - using SceneCameraMotionAnalyzer::DecideCameraMotionType; - using SceneCameraMotionAnalyzer::PopulateFocusPointFrames; -}; - -// Checks that DecideCameraMotionType checks that output pointers are not null. -TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeChecksOutputNotNull) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions(); - SceneKeyFrameCropSummary scene_summary; - SceneCameraMotion camera_motion; - auto status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec, - 0, nullptr, &camera_motion); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Scene summary is null.")); - status = analyzer.DecideCameraMotionType(crop_options, kSceneTimeSpanSec, 0, - &scene_summary, nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Scene camera motion is null.")); -} - -// Checks that DecideCameraMotionType properly handles the case where no key -// frame has any focus region, and sets the camera motion type to steady and -// the look-at position to the scene frame center. -TEST(SceneCameraMotionAnalyzerTest, - DecideCameraMotionTypeWithoutAnyFocusRegion) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions(); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_has_salient_region(false); - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType( - crop_options, kSceneTimeSpanSec, 0, &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_steady_motion()); - const auto& steady_motion = camera_motion.steady_motion(); - EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_x(), - kSceneFrameWidth / 2.0f); - EXPECT_FLOAT_EQ(steady_motion.steady_look_at_center_y(), - kSceneFrameHeight / 2.0f); -} - -// Checks that DecideCameraMotionType properly handles the camera sweeps from -// left to right. -TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingLeftToRight) { - SceneCameraMotionAnalyzerOptions options; - options.set_sweep_entire_frame(true); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_frame_success_rate( - options.minimum_success_rate_for_sweeping() / 2.0f); - scene_summary.set_crop_window_width(kTargetWidth * 1.5f); // horizontal sweep - scene_summary.set_crop_window_height(kTargetHeight); - const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - time_span, 0, &scene_summary, - &camera_motion)); - - EXPECT_TRUE(camera_motion.has_sweeping_motion()); - const auto& sweeping_motion = camera_motion.sweeping_motion(); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(), 0.0f); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(), - static_cast(kSceneFrameWidth)); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(), - kSceneFrameHeight / 2.0f); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(), - kSceneFrameHeight / 2.0f); -} - -// Checks that DecideCameraMotionType properly handles the camera sweeps from -// top to bottom. -TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingTopToBottom) { - SceneCameraMotionAnalyzerOptions options; - options.set_sweep_entire_frame(true); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_frame_success_rate( - options.minimum_success_rate_for_sweeping() / 2.0f); - scene_summary.set_crop_window_width(kTargetWidth); - scene_summary.set_crop_window_height(kTargetHeight * 1.5f); // vertical sweep - const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - time_span, 0, &scene_summary, - &camera_motion)); - - EXPECT_TRUE(camera_motion.has_sweeping_motion()); - const auto& sweeping_motion = camera_motion.sweeping_motion(); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(), 0.0f); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(), - static_cast(kSceneFrameWidth)); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(), - kSceneFrameWidth / 2.0f); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(), - kSceneFrameWidth / 2.0f); -} - -// Checks that DecideCameraMotionType properly handles the camera sweeps from -// one corner of the center range to another. -TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeSweepingCenterRange) { - SceneCameraMotionAnalyzerOptions options; - options.set_sweep_entire_frame(false); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_frame_success_rate( - options.minimum_success_rate_for_sweeping() / 2.0f); - scene_summary.set_crop_window_width(kTargetWidth * 1.5f); - scene_summary.set_crop_window_height(kTargetHeight * 1.5f); - const double time_span = options.minimum_scene_span_sec_for_sweeping() * 2.0; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - time_span, 0, &scene_summary, - &camera_motion)); - - EXPECT_TRUE(camera_motion.has_sweeping_motion()); - const auto& sweeping_motion = camera_motion.sweeping_motion(); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_x(), - scene_summary.key_frame_center_min_x()); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_start_center_y(), - scene_summary.key_frame_center_min_y()); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_x(), - scene_summary.key_frame_center_max_x()); - EXPECT_FLOAT_EQ(sweeping_motion.sweep_end_center_y(), - scene_summary.key_frame_center_max_y()); -} - -// Checks that DecideCameraMotionType properly handles the case where motion is -// small and there are no required focus regions. -TEST(SceneCameraMotionAnalyzerTest, - DecideCameraMotionTypeSmallMotionNoRequiredFocusRegion) { - SceneCameraMotionAnalyzerOptions options; - options.set_motion_stabilization_threshold_percent(0.1); - options.set_snap_center_max_distance_percent(0.0); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options); - scene_summary.set_has_required_salient_region(false); - const int crop_region_center_x = 50; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - kSceneTimeSpanSec, 0, - &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_steady_motion()); - EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(), - crop_region_center_x); - EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); -} - -// Checks that DecideCameraMotionType properly handles the case where motion is -// small and there are required focus regions that fit in target size. -TEST(SceneCameraMotionAnalyzerTest, - DecideCameraMotionTypeSmallMotionRequiredFocusRegionInTargetSize) { - SceneCameraMotionAnalyzerOptions options; - options.set_motion_stabilization_threshold_percent(0.1); - options.set_snap_center_max_distance_percent(0.0); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options); - scene_summary.set_has_required_salient_region(true); - *scene_summary.mutable_key_frame_required_crop_region_union() = - MakeRect(40, 0, 40, 10); - const int required_region_center_x = 60; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - kSceneTimeSpanSec, 0, - &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_steady_motion()); - EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(), - required_region_center_x); - EXPECT_EQ(scene_summary.crop_window_width(), 50); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); -} - -// Checks that DecideCameraMotionType properly handles the case where motion is -// small and there are required focus regions that exceed target size. -TEST(SceneCameraMotionAnalyzerTest, - DecideCameraMotionTypeSmallMotionRequiredFocusRegionExceedingTargetSize) { - SceneCameraMotionAnalyzerOptions options; - options.set_motion_stabilization_threshold_percent(0.1); - options.set_snap_center_max_distance_percent(0.0); - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options); - scene_summary.set_has_required_salient_region(true); - *scene_summary.mutable_key_frame_required_crop_region_union() = - MakeRect(20, 0, 70, 10); - const int required_region_center_x = 55; - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - kSceneTimeSpanSec, 0, - &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_steady_motion()); - EXPECT_EQ(camera_motion.steady_motion().steady_look_at_center_x(), - required_region_center_x); - EXPECT_EQ(scene_summary.crop_window_width(), 70); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); -} - -// Checks that DecideCameraMotionType properly handles the case where motion is -// small and the middle of the key frame crop center range is close to the scene -// frame center. -TEST(SceneCameraMotionAnalyzerTest, - DecideCameraMotionTypeSmallMotionCloseToCenter) { - SceneCameraMotionAnalyzerOptions options; - options.set_motion_stabilization_threshold_percent(0.1); - options.set_snap_center_max_distance_percent(0.1); - TestableSceneCameraMotionAnalyzer analyzer(options); - KeyFrameCropOptions crop_options = GetDefaultKeyFrameCropOptions(); - const float frame_center_x = kSceneFrameWidth / 2.0f; - auto scene_summary = GetSceneKeyFrameCropSummaryWithSmallMotion(options); - scene_summary.set_key_frame_center_min_x(frame_center_x - 2); - scene_summary.set_key_frame_center_max_x(frame_center_x); - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType( - crop_options, kSceneTimeSpanSec, 0, &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_steady_motion()); - EXPECT_FLOAT_EQ(camera_motion.steady_motion().steady_look_at_center_x(), - frame_center_x); -} - -// Checks that DecideCameraMotionType properly handles the case where motion is -// not small, and sets the camera motion type to tracking. -TEST(SceneCameraMotionAnalyzerTest, DecideCameraMotionTypeTracking) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_horizontal_motion_amount( - options.motion_stabilization_threshold_percent() * 2.0); - SceneCameraMotion camera_motion; - - MP_EXPECT_OK(analyzer.DecideCameraMotionType(GetDefaultKeyFrameCropOptions(), - kSceneTimeSpanSec, 0, - &scene_summary, &camera_motion)); - EXPECT_TRUE(camera_motion.has_tracking_motion()); -} - -// Checks that PopulateFocusPointFrames checks output pointer is not null. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesChecksOutputNotNull) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - const auto status = analyzer.PopulateFocusPointFrames( - GetDefaultSceneKeyFrameCropSummary(), camera_motion, - GetDefaultSceneFrameTimestamps(), nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Output vector of FocusPointFrame is null.")); -} - -// Checks that PopulateFocusPointFrames checks scene frames size. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesChecksSceneFramesSize) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - std::vector scene_frame_timestamps(0); - std::vector focus_point_frames; - - const auto status = analyzer.PopulateFocusPointFrames( - GetDefaultSceneKeyFrameCropSummary(), camera_motion, - scene_frame_timestamps, &focus_point_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("No scene frames.")); -} - -// Checks that PopulateFocusPointFrames handles the case of no key frames. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesHandlesNoKeyFrames) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - camera_motion.mutable_steady_motion(); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_num_key_frames(0); - scene_summary.clear_key_frame_compact_infos(); - scene_summary.set_has_salient_region(false); - std::vector focus_point_frames; - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames)); -} - -// Checks that PopulateFocusPointFrames checks KeyFrameCompactInfos has the -// right size. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesChecksKeyFrameCompactInfosSize) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_num_key_frames(2 * kNumKeyFrames); - std::vector focus_point_frames; - - const auto status = analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Key frame compact infos has wrong size")); -} - -// Checks that PopulateFocusPointFrames checks SceneKeyFrameCropSummary has -// valid scene frame size. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesChecksSceneFrameSize) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_scene_frame_height(0); - std::vector focus_point_frames; - - const auto status = analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Non-positive frame height.")); -} - -// Checks that PopulateFocusPointFrames checks camera motion type is valid. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesChecksCameraMotionType) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - camera_motion.clear_motion_type(); - std::vector focus_point_frames; - - const auto status = analyzer.PopulateFocusPointFrames( - GetDefaultSceneKeyFrameCropSummary(), camera_motion, - GetDefaultSceneFrameTimestamps(), &focus_point_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Unknown motion type.")); -} - -// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when -// camera motion type is steady. -TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSteady) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto* steady_motion = camera_motion.mutable_steady_motion(); - steady_motion->set_steady_look_at_center_x(40.5); - steady_motion->set_steady_look_at_center_y(25); - std::vector focus_point_frames; - - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - GetDefaultSceneKeyFrameCropSummary(), camera_motion, - GetDefaultSceneFrameTimestamps(), &focus_point_frames)); - - EXPECT_EQ(kNumSceneFrames, focus_point_frames.size()); - for (int i = 0; i < kNumSceneFrames; ++i) { - // FocusPointFrameType is CENTER. - EXPECT_EQ(focus_point_frames[i].point_size(), 1); - const auto& point = focus_point_frames[i].point(0); - EXPECT_FLOAT_EQ( - point.norm_point_x(), - steady_motion->steady_look_at_center_x() / kSceneFrameWidth); - EXPECT_FLOAT_EQ( - point.norm_point_y(), - steady_motion->steady_look_at_center_y() / kSceneFrameHeight); - EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight()); - } -} - -// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when -// FocusPointFrameType is TOPMOST_AND_BOTTOMMOST. -TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesTopAndBottom) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto* steady_motion = camera_motion.mutable_steady_motion(); - steady_motion->set_steady_look_at_center_x(40.5); - steady_motion->set_steady_look_at_center_y(25); - // Forces FocusPointFrameType to be TOPMOST_AND_BOTTOMOST. - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_crop_window_height(kSceneFrameHeight); - std::vector focus_point_frames; - - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames)); - - EXPECT_EQ(kNumSceneFrames, focus_point_frames.size()); - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(focus_point_frames[i].point_size(), 2); - const auto& point1 = focus_point_frames[i].point(0); - const auto& point2 = focus_point_frames[i].point(1); - EXPECT_FLOAT_EQ( - point1.norm_point_x(), - steady_motion->steady_look_at_center_x() / kSceneFrameWidth); - EXPECT_FLOAT_EQ(point1.norm_point_y(), 0.0f); - EXPECT_FLOAT_EQ( - point2.norm_point_x(), - steady_motion->steady_look_at_center_x() / kSceneFrameWidth); - EXPECT_FLOAT_EQ(point2.norm_point_y(), 1.0f); - EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight()); - EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight()); - } -} - -// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when -// FocusPointFrameType is LEFTMOST_AND_RIGHTMOST. -TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesLeftAndRight) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto* steady_motion = camera_motion.mutable_steady_motion(); - steady_motion->set_steady_look_at_center_x(40.5); - steady_motion->set_steady_look_at_center_y(25); - // Forces FocusPointFrameType to be LEFTMOST_AND_RIGHTMOST. - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_crop_window_width(kSceneFrameWidth); - std::vector focus_point_frames; - - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames)); - - EXPECT_EQ(kNumSceneFrames, focus_point_frames.size()); - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(focus_point_frames[i].point_size(), 2); - const auto& point1 = focus_point_frames[i].point(0); - const auto& point2 = focus_point_frames[i].point(1); - EXPECT_FLOAT_EQ(point1.norm_point_x(), 0.0f); - EXPECT_FLOAT_EQ( - point1.norm_point_y(), - steady_motion->steady_look_at_center_y() / kSceneFrameHeight); - EXPECT_FLOAT_EQ(point2.norm_point_x(), 1.0f); - EXPECT_FLOAT_EQ( - point1.norm_point_y(), - steady_motion->steady_look_at_center_y() / kSceneFrameHeight); - EXPECT_FLOAT_EQ(point1.weight(), options.maximum_salient_point_weight()); - EXPECT_FLOAT_EQ(point2.weight(), options.maximum_salient_point_weight()); - } -} - -// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when -// camera motion type is sweeping. -TEST(SceneCameraMotionAnalyzerTest, PopulateFocusPointFramesSweeping) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - auto* sweeping_motion = camera_motion.mutable_sweeping_motion(); - sweeping_motion->set_sweep_start_center_x(5); - sweeping_motion->set_sweep_start_center_y(50); - sweeping_motion->set_sweep_end_center_x(95); - sweeping_motion->set_sweep_end_center_y(50); - const int num_frames = 10; - const std::vector positions_x = {5, 15, 25, 35, 45, - 55, 65, 75, 85, 95}; - const std::vector positions_y = {50, 50, 50, 50, 50, - 50, 50, 50, 50, 50}; - std::vector scene_frame_timestamps(num_frames); - std::iota(scene_frame_timestamps.begin(), scene_frame_timestamps.end(), 0); - std::vector focus_point_frames; - - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - GetDefaultSceneKeyFrameCropSummary(), camera_motion, - scene_frame_timestamps, &focus_point_frames)); - - EXPECT_EQ(num_frames, focus_point_frames.size()); - for (int i = 0; i < num_frames; ++i) { - EXPECT_EQ(focus_point_frames[i].point_size(), 1); - const auto& point = focus_point_frames[i].point(0); - EXPECT_FLOAT_EQ(positions_x[i] / kSceneFrameWidth, point.norm_point_x()); - EXPECT_FLOAT_EQ(positions_y[i] / kSceneFrameHeight, point.norm_point_y()); - } -} - -// Checks that PopulateFocusPointFrames checks tracking handles the case when -// maximum score is 0. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesTrackingHandlesZeroScore) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - camera_motion.mutable_tracking_motion(); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_key_frame_max_score(0.0); - for (int i = 0; i < kNumKeyFrames; ++i) { - scene_summary.mutable_key_frame_compact_infos(i)->set_score(0.0); - } - std::vector focus_point_frames; - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames)); -} - -// Checks that PopulateFocusPointFrames skips empty key frames when camera -// motion type is tracking. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesTrackingSkipsEmptyKeyFrames) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - camera_motion.mutable_tracking_motion(); - SceneKeyFrameCropSummary scene_summary; - scene_summary.set_scene_frame_width(kSceneFrameWidth); - scene_summary.set_scene_frame_height(kSceneFrameHeight); - scene_summary.set_num_key_frames(2); - - // Sets first key frame to be empty and second frame to be normal. - const float center_x = 25.0f, center_y = 25.0f; - auto* first_frame_compact_info = scene_summary.add_key_frame_compact_infos(); - first_frame_compact_info->set_center_x(-1.0); - auto* second_frame_compact_info = scene_summary.add_key_frame_compact_infos(); - second_frame_compact_info->set_center_x(center_x); - second_frame_compact_info->set_center_y(center_y); - second_frame_compact_info->set_score(1.0); - scene_summary.set_key_frame_center_min_x(center_x); - scene_summary.set_key_frame_center_max_x(center_x); - scene_summary.set_key_frame_center_min_y(center_y); - scene_summary.set_key_frame_center_max_y(center_y); - scene_summary.set_key_frame_min_score(1.0); - scene_summary.set_key_frame_max_score(1.0); - - // Aligns timestamps of scene frames with key frames. - scene_summary.mutable_key_frame_compact_infos(0)->set_timestamp_ms(10); - scene_summary.mutable_key_frame_compact_infos(1)->set_timestamp_ms(20); - std::vector scene_frame_timestamps = {10, 20}; - - std::vector focus_point_frames; - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames(scene_summary, camera_motion, - scene_frame_timestamps, - &focus_point_frames)); - - // Both scene frames should have focus point frames based on the second key - // frame since the first one is empty and not used. - for (int i = 0; i < 2; ++i) { - EXPECT_EQ(focus_point_frames[i].point_size(), 1); - const auto& point = focus_point_frames[i].point(0); - EXPECT_FLOAT_EQ(point.norm_point_x(), center_x / kSceneFrameWidth); - EXPECT_FLOAT_EQ(point.norm_point_y(), center_y / kSceneFrameHeight); - EXPECT_FLOAT_EQ(point.weight(), options.maximum_salient_point_weight()); - } -} - -// Checks that PopulateFocusPointFrames properly sets FocusPointFrames when -// camera motion type is tracking, piecewise-linearly interpolating key frame -// centers and scores, and scaling scores so that maximum weight is equal to -// maximum_salient_point_weight. -TEST(SceneCameraMotionAnalyzerTest, - PopulateFocusPointFramesTrackingTracksKeyFrames) { - SceneCameraMotionAnalyzerOptions options; - TestableSceneCameraMotionAnalyzer analyzer(options); - SceneCameraMotion camera_motion; - camera_motion.mutable_tracking_motion(); - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - const std::vector centers_x = {14.0, 5.0, 40.0, 70.0, 30.0}; - const std::vector centers_y = {60.0, 50.0, 80.0, 0.0, 20.0}; - const std::vector scores = {0.1, 1.0, 2.0, 0.6, 0.9}; - scene_summary.set_key_frame_min_score(0.1); - scene_summary.set_key_frame_max_score(2.0); - for (int i = 0; i < kNumKeyFrames; ++i) { - auto* compact_info = scene_summary.mutable_key_frame_compact_infos(i); - compact_info->set_center_x(centers_x[i]); - compact_info->set_center_y(centers_y[i]); - compact_info->set_score(scores[i]); - } - - // Get reference scene frame results from csv file. - const std::string scene_frame_results_file_path = - mediapipe::file::JoinPath("./", kCameraTrackingSceneFrameResultsFile); - std::string csv_file_content; - MP_ASSERT_OK(mediapipe::file::GetContents(scene_frame_results_file_path, - &csv_file_content)); - std::vector lines = absl::StrSplit(csv_file_content, '\n'); - std::vector records; - for (const auto& line : lines) { - std::vector r = absl::StrSplit(line, ','); - records.insert(records.end(), r.begin(), r.end()); - } - CHECK_EQ(records.size(), kNumSceneFrames * 3 + 1); - - std::vector focus_point_frames; - MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( - scene_summary, camera_motion, GetDefaultSceneFrameTimestamps(), - &focus_point_frames)); - - float max_weight = 0.0; - const float tolerance = 1e-4; - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(focus_point_frames[i].point_size(), 1); - const auto& point = focus_point_frames[i].point(0); - const float expected_x = std::stof(records[i * 3]); - const float expected_y = std::stof(records[i * 3 + 1]); - const float expected_weight = std::stof(records[i * 3 + 2]); - EXPECT_LE(std::fabs(point.norm_point_x() - expected_x), tolerance); - EXPECT_LE(std::fabs(point.norm_point_y() - expected_y), tolerance); - EXPECT_LE(std::fabs(point.weight() - expected_weight), tolerance); - max_weight = std::max(max_weight, point.weight()); - } - EXPECT_LE(std::fabs(max_weight - options.maximum_salient_point_weight()), - tolerance); -} - -// Checks that AnalyzeSceneAndPopulateFocusPointFrames analyzes scene and -// populates focus point frames. -TEST(SceneCameraMotionAnalyzerTest, AnalyzeSceneAndPopulateFocusPointFrames) { - SceneCameraMotionAnalyzerOptions options; - SceneCameraMotionAnalyzer analyzer(options); - SceneKeyFrameCropSummary scene_summary; - std::vector focus_point_frames; - - MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kSceneFrameWidth, kSceneFrameHeight, GetDefaultSceneFrameTimestamps(), - false, &scene_summary, &focus_point_frames)); - EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames); - EXPECT_EQ(focus_point_frames.size(), kNumSceneFrames); -} - -// Checks that AnalyzeSceneAndPopulateFocusPointFrames optionally returns -// scene camera motion. -TEST(SceneCameraMotionAnalyzerTest, - AnalyzeSceneAndPopulateFocusPointFramesReturnsSceneCameraMotion) { - SceneCameraMotionAnalyzerOptions options; - SceneCameraMotionAnalyzer analyzer(options); - SceneKeyFrameCropSummary scene_summary; - std::vector focus_point_frames; - SceneCameraMotion scene_camera_motion; - - MP_EXPECT_OK(analyzer.AnalyzeSceneAndPopulateFocusPointFrames( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kSceneFrameWidth, kSceneFrameHeight, GetDefaultSceneFrameTimestamps(), - false, &scene_summary, &focus_point_frames, &scene_camera_motion)); - EXPECT_TRUE(scene_camera_motion.has_steady_motion()); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc b/mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc deleted file mode 100644 index a3c6f17c6..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.cc +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h" - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h" -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -// TODO: Parameterize FOV based on camera specs. -constexpr float kWidthFieldOfView = 60; - -namespace mediapipe { -namespace autoflip { - -absl::Status SceneCropper::ProcessKinematicPathSolver( - const SceneKeyFrameCropSummary& scene_summary, - const std::vector& scene_timestamps, - const std::vector& is_key_frames, - const std::vector& focus_point_frames, - const bool continue_last_scene, std::vector* all_xforms) { - // TODO: Fix upstream calculators to not crop beyond portrait target - // value. - /* - RET_CHECK(scene_summary.scene_frame_height() == - scene_summary.crop_window_height()) - << "Kinematic path solver does not yet support horizontal cropping."; - */ - - RET_CHECK(scene_timestamps.size() == focus_point_frames.size()) - << "Kinematic path solver does not yet support downsampled detections."; - - if (!path_solver_initalized_ || !continue_last_scene) { - int min_location = scene_summary.crop_window_width() / 2; - int max_location = scene_summary.scene_frame_width() - - scene_summary.crop_window_width() / 2; - kinematic_path_solver_ = std::make_unique( - camera_motion_options_.kinematic_options(), min_location, max_location, - static_cast(frame_width_) / kWidthFieldOfView); - path_solver_initalized_ = true; - } - int keyframe_counter = 0; - for (int i = 0; i < is_key_frames.size(); i++) { - if (is_key_frames[i]) { - RET_CHECK_EQ(focus_point_frames[keyframe_counter].point().size(), 2) - << "Expected focus_points to equal 2"; - int observed_x = std::round( - focus_point_frames[keyframe_counter].point(0).norm_point_x() * - scene_summary.scene_frame_width()); - MP_RETURN_IF_ERROR(kinematic_path_solver_->AddObservation( - observed_x, scene_timestamps[i])); - keyframe_counter++; - } else { - MP_RETURN_IF_ERROR( - kinematic_path_solver_->UpdatePrediction(scene_timestamps[i])); - } - int x_path; - MP_RETURN_IF_ERROR(kinematic_path_solver_->GetState(&x_path)); - cv::Mat transform = cv::Mat::eye(2, 3, CV_32FC1); - transform.at(0, 2) = - -(x_path - scene_summary.crop_window_width() / 2); - all_xforms->push_back(transform); - } - return absl::OkStatus(); -} - -absl::Status SceneCropper::CropFrames( - const SceneKeyFrameCropSummary& scene_summary, - const std::vector& scene_timestamps, - const std::vector& is_key_frames, - const std::vector& scene_frames_or_empty, - const std::vector& focus_point_frames, - const std::vector& prior_focus_point_frames, - int top_static_border_size, int bottom_static_border_size, - const bool continue_last_scene, std::vector* crop_from_location, - std::vector* cropped_frames) { - const int num_scene_frames = scene_timestamps.size(); - RET_CHECK_GT(num_scene_frames, 0) << "No scene frames."; - RET_CHECK_EQ(focus_point_frames.size(), num_scene_frames) - << "Wrong size of FocusPointFrames."; - - const int frame_width = scene_summary.scene_frame_width(); - const int frame_height = scene_summary.scene_frame_height(); - const int crop_width = scene_summary.crop_window_width(); - const int crop_height = scene_summary.crop_window_height(); - RET_CHECK_GT(crop_width, 0) << "Crop width is non-positive."; - RET_CHECK_GT(crop_height, 0) << "Crop height is non-positive."; - RET_CHECK_LE(crop_width, frame_width) << "Crop width exceeds frame width."; - RET_CHECK_LE(crop_height, frame_height) - << "Crop height exceeds frame height."; - - RET_CHECK(camera_motion_options_.has_polynomial_path_solver() || - camera_motion_options_.has_kinematic_options()) - << "No camera motion model selected."; - - // Computes transforms. - - std::vector scene_frame_xforms; - int num_prior = 0; - if (camera_motion_options_.has_polynomial_path_solver()) { - num_prior = prior_focus_point_frames.size(); - std::vector all_xforms; - PolynomialRegressionPathSolver solver; - RET_CHECK_OK(solver.ComputeCameraPath( - focus_point_frames, prior_focus_point_frames, frame_width, frame_height, - crop_width, crop_height, &all_xforms)); - - scene_frame_xforms = - std::vector(all_xforms.begin() + num_prior, all_xforms.end()); - - // Convert the matrix from center-aligned to upper-left aligned. - for (cv::Mat& xform : scene_frame_xforms) { - cv::Mat affine_opencv = cv::Mat::eye(2, 3, CV_32FC1); - affine_opencv.at(0, 2) = - -(xform.at(0, 2) + frame_width / 2 - crop_width / 2); - affine_opencv.at(1, 2) = - -(xform.at(1, 2) + frame_height / 2 - crop_height / 2); - xform = affine_opencv; - } - } else if (camera_motion_options_.has_kinematic_options()) { - num_prior = 0; - MP_RETURN_IF_ERROR(ProcessKinematicPathSolver( - scene_summary, scene_timestamps, is_key_frames, focus_point_frames, - continue_last_scene, &scene_frame_xforms)); - } - - // Store the "crop from" location on the input frame for use with an external - // renderer. - for (int i = 0; i < num_scene_frames; i++) { - const int left = -(scene_frame_xforms[i].at(0, 2)); - const int top = - top_static_border_size - (scene_frame_xforms[i].at(1, 2)); - crop_from_location->push_back(cv::Rect(left, top, crop_width, crop_height)); - } - - // If no cropped_frames is passed in, return directly. - if (!cropped_frames) { - return absl::OkStatus(); - } - RET_CHECK(!scene_frames_or_empty.empty()) - << "If |cropped_frames| != nullptr, scene_frames_or_empty must not be " - "empty."; - // Prepares cropped frames. - cropped_frames->resize(num_scene_frames); - for (int i = 0; i < num_scene_frames; ++i) { - (*cropped_frames)[i] = cv::Mat::zeros(crop_height, crop_width, - scene_frames_or_empty[i].type()); - } - return AffineRetarget(cv::Size(crop_width, crop_height), - scene_frames_or_empty, scene_frame_xforms, - cropped_frames); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h b/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h deleted file mode 100644 index 0e5c332db..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_ - -#include -#include - -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// This class is a thin wrapper around the Retargeter class to crop a collection -// of scene frames given SceneKeyFrameCropSummary and their FocusPointFrames. -// -// Upstream inputs: -// - SceneKeyFrameCropSummary scene_summary. -// - std::vector focus_point_frames. -// - std::vector prior_focus_point_frames. -// - std::vector scene_frames; -// -// Example usage: -// SceneCropperOptions scene_cropper_options; -// SceneCropper scene_cropper(scene_cropper_options); -// std::vector cropped_frames; -// CHECK_OK(scene_cropper.CropFrames( -// scene_summary, scene_frames, focus_point_frames, -// prior_focus_point_frames, &cropped_frames)); -class SceneCropper { - public: - SceneCropper(const CameraMotionOptions& camera_motion_options, - const int frame_width, const int frame_height) - : path_solver_initalized_(false), - camera_motion_options_(camera_motion_options), - frame_width_(frame_width), - frame_height_(frame_height) {} - ~SceneCropper() {} - - // Computes transformation matrix given SceneKeyFrameCropSummary, - // FocusPointFrames, and any prior FocusPointFrames (to ensure smoothness when - // there was no actual scene change). Optionally crops the input frames based - // on the transform matrix if |cropped_frames| is not nullptr and - // |scene_frames_or_empty| isn't empty. - // TODO: split this function into two separate functions. - absl::Status CropFrames( - const SceneKeyFrameCropSummary& scene_summary, - const std::vector& scene_timestamps, - const std::vector& is_key_frames, - const std::vector& scene_frames_or_empty, - const std::vector& focus_point_frames, - const std::vector& prior_focus_point_frames, - int top_static_border_size, int bottom_static_border_size, - const bool continue_last_scene, std::vector* crop_from_location, - std::vector* cropped_frames); - - absl::Status ProcessKinematicPathSolver( - const SceneKeyFrameCropSummary& scene_summary, - const std::vector& scene_timestamps, - const std::vector& is_key_frames, - const std::vector& focus_point_frames, - const bool continue_last_scene, std::vector* all_xforms); - - private: - bool path_solver_initalized_; - std::unique_ptr kinematic_path_solver_; - CameraMotionOptions camera_motion_options_; - int frame_width_; - int frame_height_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc b/mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc deleted file mode 100644 index fb2c989b2..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropper_test.cc +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/scene_cropper.h" - -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { - -using testing::HasSubstr; - -const int kCropWidth = 90; -const int kCropHeight = 160; - -const int kSceneWidth = 320; -const int kSceneHeight = 180; - -const int kNumSceneFrames = 30; - -// Returns default values for SceneKeyFrameCropSummary. Sets scene size and crop -// window size from default values. -SceneKeyFrameCropSummary GetDefaultSceneKeyFrameCropSummary() { - SceneKeyFrameCropSummary scene_summary; - scene_summary.set_scene_frame_width(kSceneWidth); - scene_summary.set_scene_frame_height(kSceneHeight); - scene_summary.set_crop_window_width(kCropWidth); - scene_summary.set_crop_window_height(kCropHeight); - return scene_summary; -} - -// Returns default values for scene frames of size kNumSceneFrames. Stes each -// frame to be solid red color at default scene size. -std::vector GetDefaultSceneFrames() { - std::vector scene_frames(kNumSceneFrames); - for (int i = 0; i < kNumSceneFrames; ++i) { - scene_frames[i] = cv::Mat(kSceneHeight, kSceneWidth, CV_8UC3); - scene_frames[i] = cv::Scalar(255, 0, 0); - } - return scene_frames; -} - -// Makes a vector of FocusPointFrames given size. Stes each FocusPointFrame -// to have one FocusPoint at the center of the frame. -std::vector GetFocusPointFrames(const int num_frames) { - std::vector focus_point_frames(num_frames); - for (int i = 0; i < num_frames; ++i) { - auto* point = focus_point_frames[i].add_point(); - point->set_norm_point_x(0.5); - point->set_norm_point_y(0.5); - } - return focus_point_frames; -} -// Returns default values for FocusPointFrames of size kNumSceneFrames. -std::vector GetDefaultFocusPointFrames() { - return GetFocusPointFrames(kNumSceneFrames); -} - -std::vector GetTimestamps(const int num_frames) { - std::vector timestamps; - for (int i = 0; i < num_frames; ++i) { - timestamps.push_back(i * 100000); - } - return timestamps; -} - -std::vector GetIsKeyframe(const int num_frames) { - std::vector is_keyframe; - for (int i = 0; i < num_frames; ++i) { - is_keyframe.push_back(false); - } - return is_keyframe; -} - -// Checks that CropFrames checks that scene frames size is positive. -TEST(SceneCropperTest, CropFramesChecksSceneFramesSize) { - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector scene_frames(0); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto status = scene_cropper.CropFrames( - GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(scene_frames.size()), - GetIsKeyframe(scene_frames.size()), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false, - &crop_from_locations, &cropped_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("No scene frames.")); -} - -// Checks that CropFrames checks that FocusPointFrames has the right size. - -TEST(SceneCropperTest, CropFramesChecksFocusPointFramesSize) { - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - const auto status = scene_cropper.CropFrames( - GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(kNumSceneFrames), - GetIsKeyframe(kNumSceneFrames), scene_frames, - GetFocusPointFrames(kNumSceneFrames - 1), GetFocusPointFrames(0), 0, 0, - false, &crop_from_locations, &cropped_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Wrong size of FocusPointFrames")); -} - -// Checks that CropFrames checks crop size is positive. -TEST(SceneCropperTest, CropFramesChecksCropSizePositive) { - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_crop_window_width(-1); - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - const auto status = scene_cropper.CropFrames( - scene_summary, GetTimestamps(kNumSceneFrames), - GetIsKeyframe(kNumSceneFrames), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false, - &crop_from_locations, &cropped_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Crop width is non-positive.")); -} - -// Checks that CropFrames checks that crop size does not exceed frame size. -TEST(SceneCropperTest, InitializesRetargeterChecksCropSizeNotExceedFrameSize) { - auto scene_summary = GetDefaultSceneKeyFrameCropSummary(); - scene_summary.set_crop_window_height(kSceneHeight + 1); - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - const auto status = scene_cropper.CropFrames( - scene_summary, GetTimestamps(kNumSceneFrames), - GetIsKeyframe(kNumSceneFrames), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false, - &crop_from_locations, &cropped_frames); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Crop height exceeds frame height.")); -} - -// Checks that CropFrames works when there are not any prior FocusPointFrames. -TEST(SceneCropperTest, CropFramesWorksWithoutPriorFocusPointFrames) { - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - MP_ASSERT_OK(scene_cropper.CropFrames( - GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(kNumSceneFrames), - GetIsKeyframe(kNumSceneFrames), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(0), 0, 0, false, - &crop_from_locations, &cropped_frames)); - ASSERT_EQ(cropped_frames.size(), kNumSceneFrames); - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(cropped_frames[i].rows, kCropHeight); - EXPECT_EQ(cropped_frames[i].cols, kCropWidth); - } -} - -// Checks that CropFrames works when there are prior FocusPointFrames. -TEST(SceneCropperTest, CropFramesWorksWithPriorFocusPointFrames) { - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - MP_EXPECT_OK(scene_cropper.CropFrames( - GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(scene_frames.size()), - GetIsKeyframe(scene_frames.size()), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0, false, - &crop_from_locations, &cropped_frames)); - EXPECT_EQ(cropped_frames.size(), kNumSceneFrames); - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(cropped_frames[i].rows, kCropHeight); - EXPECT_EQ(cropped_frames[i].cols, kCropWidth); - } -} - -// Checks that crop_from_locations gets the correct results. -TEST(SceneCropperTest, CropFromLocation) { - CameraMotionOptions options; - options.mutable_polynomial_path_solver()->set_prior_frame_buffer_size(30); - SceneCropper scene_cropper(options, kSceneWidth, kSceneHeight); - std::vector cropped_frames; - std::vector crop_from_locations; - const auto& scene_frames = GetDefaultSceneFrames(); - MP_EXPECT_OK(scene_cropper.CropFrames( - GetDefaultSceneKeyFrameCropSummary(), GetTimestamps(scene_frames.size()), - GetIsKeyframe(scene_frames.size()), scene_frames, - GetDefaultFocusPointFrames(), GetFocusPointFrames(3), 0, 0, false, - &crop_from_locations, &cropped_frames)); - EXPECT_EQ(cropped_frames.size(), kNumSceneFrames); - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(cropped_frames[i].rows, kCropHeight); - EXPECT_EQ(cropped_frames[i].cols, kCropWidth); - } - for (int i = 0; i < kNumSceneFrames; ++i) { - EXPECT_EQ(crop_from_locations[i].height, kCropHeight); - EXPECT_EQ(crop_from_locations[i].width, kCropWidth); - } -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc b/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc deleted file mode 100644 index d27423c9a..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h" - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// Colors for focus signal sources. -const cv::Scalar kCyan = - cv::Scalar(0.0, 255.0, 255.0); // brain object detector -const cv::Scalar kMagenta = cv::Scalar(255.0, 0.0, 255.0); // motion -const cv::Scalar kYellow = cv::Scalar(255.0, 255.0, 0.0); // fg ocr -const cv::Scalar kLightYellow = cv::Scalar(255.0, 250.0, 205.0); // bg ocr -const cv::Scalar kRed = cv::Scalar(255.0, 0.0, 0.0); // logo -const cv::Scalar kGreen = cv::Scalar(0.0, 255.0, 0.0); // face -const cv::Scalar kBlue = - cv::Scalar(0.0, 0.0, 255.0); // creatism saliency model -const cv::Scalar kOrange = - cv::Scalar(255.0, 165.0, 0.0); // ica object detector -const cv::Scalar kWhite = cv::Scalar(255.0, 255.0, 255.0); // others - -absl::Status DrawDetectionsAndCropRegions( - const std::vector& scene_frames, - const std::vector& is_key_frames, - const std::vector& key_frame_infos, - const std::vector& key_frame_crop_results, - const ImageFormat::Format image_format, - std::vector>* viz_frames) { - RET_CHECK(viz_frames) << "Output viz frames is null."; - viz_frames->clear(); - const int num_frames = scene_frames.size(); - - std::pair crop_corners; - std::vector> region_corners; - std::vector region_colors; - auto RectToCvPoints = - [](const Rect& rect) -> std::pair { - return std::make_pair( - cv::Point(rect.x(), rect.y()), - cv::Point(rect.x() + rect.width(), rect.y() + rect.height())); - }; - - int key_frame_idx = 0; - for (int i = 0; i < num_frames; ++i) { - const auto& scene_frame = scene_frames[i]; - auto viz_frame = absl::make_unique( - image_format, scene_frame.cols, scene_frame.rows); - cv::Mat viz_mat = formats::MatView(viz_frame.get()); - scene_frame.copyTo(viz_mat); - - if (is_key_frames[i]) { - const auto& bbox = key_frame_crop_results[key_frame_idx].region(); - crop_corners = RectToCvPoints(bbox); - region_corners.clear(); - region_colors.clear(); - const auto& detections = key_frame_infos[key_frame_idx].detections(); - for (int j = 0; j < detections.detections_size(); ++j) { - const auto& detection = detections.detections(j); - const auto corners = RectToCvPoints(detection.location()); - region_corners.push_back(corners); - if (detection.signal_type().has_standard()) { - switch (detection.signal_type().standard()) { - case SignalType::FACE_FULL: - case SignalType::FACE_LANDMARK: - case SignalType::FACE_ALL_LANDMARKS: - case SignalType::FACE_CORE_LANDMARKS: - region_colors.push_back(kGreen); - break; - case SignalType::HUMAN: - region_colors.push_back(kLightYellow); - break; - case SignalType::CAR: - region_colors.push_back(kMagenta); - break; - case SignalType::PET: - region_colors.push_back(kYellow); - break; - case SignalType::OBJECT: - region_colors.push_back(kCyan); - break; - case SignalType::MOTION: - case SignalType::TEXT: - case SignalType::LOGO: - region_colors.push_back(kRed); - break; - case SignalType::USER_HINT: - default: - region_colors.push_back(kWhite); - break; - } - } else { - // For the case where "custom" signal type is used. - region_colors.push_back(kWhite); - } - } - key_frame_idx++; - } - - cv::rectangle(viz_mat, crop_corners.first, crop_corners.second, kGreen, 4); - for (int j = 0; j < region_corners.size(); ++j) { - cv::rectangle(viz_mat, region_corners[j].first, region_corners[j].second, - region_colors[j], 2); - } - viz_frames->push_back(std::move(viz_frame)); - } - return absl::OkStatus(); -} - -namespace { -cv::Rect LimitBounds(const cv::Rect& rect, const int max_width, - const int max_height) { - cv::Rect result; - result.x = fmax(rect.x, 0); - result.y = fmax(rect.y, 0); - result.width = - result.x + rect.width >= max_width ? max_width - result.x : rect.width; - result.height = result.y + rect.height >= max_height ? max_height - result.y - : rect.height; - return result; -} -} // namespace - -absl::Status DrawDetectionAndFramingWindow( - const std::vector& org_scene_frames, - const std::vector& crop_from_locations, - const ImageFormat::Format image_format, const float overlay_opacity, - std::vector>* viz_frames) { - for (int i = 0; i < org_scene_frames.size(); i++) { - const auto& scene_frame = org_scene_frames[i]; - auto viz_frame = absl::make_unique( - image_format, scene_frame.cols, scene_frame.rows); - cv::Mat darkened = formats::MatView(viz_frame.get()); - scene_frame.copyTo(darkened); - cv::Mat overlay = cv::Mat::zeros(darkened.size(), darkened.type()); - cv::addWeighted(overlay, overlay_opacity, darkened, 1 - overlay_opacity, 0, - darkened); - const auto& crop_from_bounded = - LimitBounds(crop_from_locations[i], scene_frame.cols, scene_frame.rows); - scene_frame(crop_from_bounded).copyTo(darkened(crop_from_bounded)); - viz_frames->push_back(std::move(viz_frame)); - } - return absl::OkStatus(); -} - -absl::Status DrawFocusPointAndCropWindow( - const std::vector& scene_frames, - const std::vector& focus_point_frames, - const float overlay_opacity, const int crop_window_width, - const int crop_window_height, const ImageFormat::Format image_format, - std::vector>* viz_frames) { - RET_CHECK(viz_frames) << "Output viz frames is null."; - viz_frames->clear(); - const int num_frames = scene_frames.size(); - RET_CHECK_GT(crop_window_width, 0) << "Crop window width is non-positive."; - RET_CHECK_GT(crop_window_height, 0) << "Crop window height is non-positive."; - const int half_width = crop_window_width / 2; - const int half_height = crop_window_height / 2; - - for (int i = 0; i < num_frames; ++i) { - const auto& scene_frame = scene_frames[i]; - auto viz_frame = absl::make_unique( - image_format, scene_frame.cols, scene_frame.rows); - cv::Mat darkened = formats::MatView(viz_frame.get()); - scene_frame.copyTo(darkened); - cv::Mat viz_mat = darkened.clone(); - - // Darken the background. - cv::Mat overlay = cv::Mat::zeros(darkened.size(), darkened.type()); - cv::addWeighted(overlay, overlay_opacity, darkened, 1 - overlay_opacity, 0, - darkened); - - if (focus_point_frames[i].point_size() > 0) { - float center_x = 0.0f, center_y = 0.0f; - for (int j = 0; j < focus_point_frames[i].point_size(); ++j) { - const auto& point = focus_point_frames[i].point(j); - const int x = point.norm_point_x() * scene_frame.cols; - const int y = point.norm_point_y() * scene_frame.rows; - cv::circle(viz_mat, cv::Point(x, y), 3, kRed, cv::FILLED); - center_x += x; - center_y += y; - } - center_x /= focus_point_frames[i].point_size(); - center_y /= focus_point_frames[i].point_size(); - cv::Point min_corner(center_x - half_width, center_y - half_height); - cv::Point max_corner(center_x + half_width, center_y + half_height); - viz_mat(cv::Rect(min_corner, max_corner)) - .copyTo(darkened(cv::Rect(min_corner, max_corner))); - } - viz_frames->push_back(std::move(viz_frame)); - } - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h b/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h deleted file mode 100644 index 01f8c5de5..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropping_viz.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// Draws the detections and crop regions on the scene frame. To make -// visualization smoother, applies piecewise-constant interpolation on non-key -// frames. This helps visualize the inputs to and outputs from the -// FrameCropRegionComputer. Uses thick green for computed crop regions. Uses -// different colors for different focus signals, faces are green, motion is -// magenta, logos are red, ocrs are yellow (foreground) and light yellow -// (background), brain objects are cyan, ica objects are orange, and the rest -// are white. -absl::Status DrawDetectionsAndCropRegions( - const std::vector& scene_frames, - const std::vector& is_key_frames, - const std::vector& key_frame_infos, - const std::vector& key_frame_crop_results, - const mediapipe::ImageFormat::Format image_format, - std::vector>* viz_frames); - -// Draws the focus point from the given FocusPointFrame and the crop window -// centered around it on the scene frame in red. This helps visualize the input -// to the retargeter. -absl::Status DrawFocusPointAndCropWindow( - const std::vector& scene_frames, - const std::vector& focus_point_frames, - const float overlay_opacity, const int crop_window_width, - const int crop_window_height, - const mediapipe::ImageFormat::Format image_format, - std::vector>* viz_frames); - -// Draws the final smoothed path of the camera retargeter by darkening the -// removed areas. -absl::Status DrawDetectionAndFramingWindow( - const std::vector& org_scene_frames, - const std::vector& crop_from_locations, - const ImageFormat::Format image_format, const float overlay_opacity, - std::vector>* viz_frames); - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_SCENE_CROPPING_VIZ_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/BUILD b/mediapipe/examples/desktop/autoflip/quality/testdata/BUILD deleted file mode 100644 index bbfc6340d..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/testdata/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -exports_files(glob(["*"])) diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv b/mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv deleted file mode 100644 index 81a57aa96..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/testdata/camera_motion_tracking_scene_frame_results.csv +++ /dev/null @@ -1,30 +0,0 @@ -0.14,0.6,5.00005 -0.125,0.583333,12.5 -0.11,0.566667,20 -0.0950004,0.55,27.5 -0.0800006,0.533334,35 -0.0650008,0.516668,42.5 -0.0500009,0.500001,50 -0.108329,0.549996,58.3333 -0.166662,0.599996,66.6667 -0.224995,0.649996,75 -0.283327,0.699995,83.3333 -0.34166,0.749995,91.6667 -0.399993,0.799994,100 -0.449994,0.666684,88.3357 -0.499993,0.533352,76.6691 -0.549993,0.40002,65.0024 -0.599992,0.266688,53.3357 -0.649992,0.133356,41.6691 -0.699991,2.40E-05,30.0024 -0.633346,0.033327,32.4999 -0.56668,0.06666,34.9999 -0.500014,0.099993,37.4999 -0.433348,0.133326,39.9998 -0.366682,0.166659,42.4998 -0.300016,0.199992,44.9999 -0.3,0.2,45.0005 -0.3,0.2,45.0005 -0.3,0.2,45.0005 -0.3,0.2,45.0005 -0.3,0.2,45.0005 diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/google.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/google.jpg deleted file mode 100644 index 25d7c1952..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/google.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3.jpg deleted file mode 100644 index 53ebcf770..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3_solid_background.jpg deleted file mode 100644 index 5d870d777..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.3_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6.jpg deleted file mode 100644 index 2ffde6739..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6_solid_background.jpg deleted file mode 100644 index ca4b2788b..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_0.6_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6.jpg deleted file mode 100644 index ad28b4154..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6_solid_background.jpg deleted file mode 100644 index 9c915c836..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.6_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.jpg deleted file mode 100644 index c03cd4610..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_1_solid_background.jpg deleted file mode 100644 index 97c0355ed..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_1_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5.jpg deleted file mode 100644 index be119b2f7..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5_solid_background.jpg deleted file mode 100644 index 2a23b16dc..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_2.5_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4.jpg deleted file mode 100644 index 5ec4ea6ec..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4_solid_background.jpg b/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4_solid_background.jpg deleted file mode 100644 index a27cb5302..000000000 Binary files a/mediapipe/examples/desktop/autoflip/quality/testdata/result_3.4_solid_background.jpg and /dev/null differ diff --git a/mediapipe/examples/desktop/autoflip/quality/utils.cc b/mediapipe/examples/desktop/autoflip/quality/utils.cc deleted file mode 100644 index 7b25930fc..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/utils.cc +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" - -#include - -#include -#include - -#include "absl/memory/memory.h" -#include "mediapipe/examples/desktop/autoflip/quality/math_utils.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -// Returns true if the first pair should be considered greater than the second. -// This is used to sort detections by scores (from high to low). -bool PairCompare(const std::pair& pair1, - const std::pair& pair2) { - return pair1.first > pair2.first; -} - -} // namespace - -template -void ScaleRect(const T& original_location, const double scale_x, - const double scale_y, Rect* scaled_location) { - scaled_location->set_x(round(original_location.x() * scale_x)); - scaled_location->set_y(round(original_location.y() * scale_y)); - scaled_location->set_width(round(original_location.width() * scale_x)); - scaled_location->set_height(round(original_location.height() * scale_y)); -} -template void ScaleRect(const Rect&, const double, const double, Rect*); -template void ScaleRect(const RectF&, const double, const double, Rect*); - -void NormalizedRectToRect(const RectF& normalized_location, const int width, - const int height, Rect* location) { - ScaleRect(normalized_location, width, height, location); -} - -absl::Status ClampRect(const int width, const int height, Rect* location) { - return ClampRect(0, 0, width, height, location); -} - -absl::Status ClampRect(const int x0, const int y0, const int x1, const int y1, - Rect* location) { - RET_CHECK(!(location->x() >= x1 || location->x() + location->width() <= x0 || - location->y() >= y1 || location->y() + location->height() <= y0)); - - int clamped_left, clamped_right, clamped_top, clamped_bottom; - RET_CHECK(MathUtil::Clamp(x0, x1, location->x(), &clamped_left)); - RET_CHECK(MathUtil::Clamp(x0, x1, location->x() + location->width(), - &clamped_right)); - RET_CHECK(MathUtil::Clamp(y0, y1, location->y(), &clamped_top)); - RET_CHECK(MathUtil::Clamp(y0, y1, location->y() + location->height(), - &clamped_bottom)); - location->set_x(clamped_left); - location->set_y(clamped_top); - location->set_width(std::max(0, clamped_right - clamped_left)); - location->set_height(std::max(0, clamped_bottom - clamped_top)); - return absl::OkStatus(); -} - -void RectUnion(const Rect& rect_to_add, Rect* rect) { - const int x1 = std::min(rect->x(), rect_to_add.x()); - const int y1 = std::min(rect->y(), rect_to_add.y()); - const int x2 = std::max(rect->x() + rect->width(), - rect_to_add.x() + rect_to_add.width()); - const int y2 = std::max(rect->y() + rect->height(), - rect_to_add.y() + rect_to_add.height()); - rect->set_x(x1); - rect->set_y(y1); - rect->set_width(x2 - x1); - rect->set_height(y2 - y1); -} - -absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms, - const DetectionSet& detections, - const int original_frame_width, - const int original_frame_height, - const int feature_frame_width, - const int feature_frame_height, - KeyFrameInfo* key_frame_info) { - RET_CHECK(key_frame_info != nullptr) << "KeyFrameInfo is null"; - RET_CHECK(original_frame_width > 0 && original_frame_height > 0 && - feature_frame_width > 0 && feature_frame_height > 0) - << "Invalid frame size."; - - const double scale_x = - static_cast(original_frame_width) / feature_frame_width; - const double scale_y = - static_cast(original_frame_height) / feature_frame_height; - - key_frame_info->set_timestamp_ms(frame_timestamp_ms); - - // Scales detections and filter out the ones with no bounding boxes. - auto* processed_detections = key_frame_info->mutable_detections(); - for (const auto& original_detection : detections.detections()) { - bool has_valid_location = true; - Rect location; - if (original_detection.has_location_normalized()) { - NormalizedRectToRect(original_detection.location_normalized(), - original_frame_width, original_frame_height, - &location); - } else if (original_detection.has_location()) { - ScaleRect(original_detection.location(), scale_x, scale_y, &location); - } else { - has_valid_location = false; - LOG(ERROR) << "Detection missing a bounding box, skipped."; - } - if (has_valid_location) { - if (!ClampRect(original_frame_width, original_frame_height, &location) - .ok()) { - LOG(ERROR) << "Invalid detection bounding box, skipped."; - continue; - } - auto* detection = processed_detections->add_detections(); - *detection = original_detection; - *(detection->mutable_location()) = location; - } - } - - return absl::OkStatus(); -} - -absl::Status SortDetections(const DetectionSet& detections, - std::vector* required_regions, - std::vector* non_required_regions) { - required_regions->clear(); - non_required_regions->clear(); - - // Makes pairs of score and index. - std::vector> required_score_idx_pairs; - std::vector> non_required_score_idx_pairs; - for (int i = 0; i < detections.detections_size(); ++i) { - const auto& detection = detections.detections(i); - const auto pair = std::make_pair(detection.score(), i); - if (detection.is_required()) { - required_score_idx_pairs.push_back(pair); - } else { - non_required_score_idx_pairs.push_back(pair); - } - } - - // Sorts required regions by score. - std::stable_sort(required_score_idx_pairs.begin(), - required_score_idx_pairs.end(), PairCompare); - for (int i = 0; i < required_score_idx_pairs.size(); ++i) { - const int original_idx = required_score_idx_pairs[i].second; - required_regions->push_back(detections.detections(original_idx)); - } - - // Sorts non-required regions by score. - std::stable_sort(non_required_score_idx_pairs.begin(), - non_required_score_idx_pairs.end(), PairCompare); - for (int i = 0; i < non_required_score_idx_pairs.size(); ++i) { - const int original_idx = non_required_score_idx_pairs[i].second; - non_required_regions->push_back(detections.detections(original_idx)); - } - - return absl::OkStatus(); -} - -absl::Status SetKeyFrameCropTarget(const int frame_width, - const int frame_height, - const double target_aspect_ratio, - KeyFrameCropOptions* crop_options) { - RET_CHECK_NE(crop_options, nullptr) << "KeyFrameCropOptions is null."; - RET_CHECK_GT(frame_width, 0) << "Frame width is non-positive."; - RET_CHECK_GT(frame_height, 0) << "Frame height is non-positive."; - RET_CHECK_GT(target_aspect_ratio, 0) - << "Target aspect ratio is non-positive."; - const double input_aspect_ratio = - static_cast(frame_width) / frame_height; - const int crop_target_width = - target_aspect_ratio < input_aspect_ratio - ? std::round(frame_height * target_aspect_ratio) - : frame_width; - const int crop_target_height = - target_aspect_ratio < input_aspect_ratio - ? frame_height - : std::round(frame_width / target_aspect_ratio); - crop_options->set_target_width(crop_target_width); - crop_options->set_target_height(crop_target_height); - return absl::OkStatus(); -} - -absl::Status AggregateKeyFrameResults( - const KeyFrameCropOptions& key_frame_crop_options, - const std::vector& key_frame_crop_results, - const int scene_frame_width, const int scene_frame_height, - SceneKeyFrameCropSummary* scene_summary) { - RET_CHECK_NE(scene_summary, nullptr) - << "Output SceneKeyFrameCropSummary is null."; - - const int num_key_frames = key_frame_crop_results.size(); - - RET_CHECK_GT(scene_frame_width, 0) << "Non-positive frame width."; - RET_CHECK_GT(scene_frame_height, 0) << "Non-positive frame height."; - - const int target_width = key_frame_crop_options.target_width(); - const int target_height = key_frame_crop_options.target_height(); - RET_CHECK_GT(target_width, 0) << "Non-positive target width."; - RET_CHECK_GT(target_height, 0) << "Non-positive target height."; - RET_CHECK_LE(target_width, scene_frame_width) - << "Target width exceeds frame width."; - RET_CHECK_LE(target_height, scene_frame_height) - << "Target height exceeds frame height."; - - scene_summary->set_scene_frame_width(scene_frame_width); - scene_summary->set_scene_frame_height(scene_frame_height); - scene_summary->set_crop_window_width(target_width); - scene_summary->set_crop_window_height(target_height); - - // Handles the corner case of no key frames. - if (num_key_frames == 0) { - scene_summary->set_has_salient_region(false); - return absl::OkStatus(); - } - - scene_summary->set_num_key_frames(num_key_frames); - scene_summary->set_key_frame_center_min_x(scene_frame_width); - scene_summary->set_key_frame_center_max_x(0); - scene_summary->set_key_frame_center_min_y(scene_frame_height); - scene_summary->set_key_frame_center_max_y(0); - scene_summary->set_key_frame_min_score(std::numeric_limits::max()); - scene_summary->set_key_frame_max_score(0.0); - - const float half_height = target_height / 2.0f; - const float half_width = target_width / 2.0f; - bool has_salient_region = false; - int num_success_frames = 0; - std::unique_ptr required_crop_region_union = nullptr; - for (int i = 0; i < num_key_frames; ++i) { - auto* key_frame_compact_info = scene_summary->add_key_frame_compact_infos(); - const auto& result = key_frame_crop_results[i]; - key_frame_compact_info->set_timestamp_ms(result.timestamp_ms()); - if (result.are_required_regions_covered_in_target_size()) { - num_success_frames++; - } - if (result.region_is_empty()) { - key_frame_compact_info->set_center_x(-1.0); - key_frame_compact_info->set_center_y(-1.0); - key_frame_compact_info->set_score(-1.0); - continue; - } - - has_salient_region = true; - if (!result.required_region_is_empty()) { - if (required_crop_region_union == nullptr) { - required_crop_region_union = - absl::make_unique(result.required_region()); - } else { - RectUnion(result.required_region(), required_crop_region_union.get()); - } - } - - const auto& region = result.region(); - float original_center_x = region.x() + region.width() / 2.0f; - float original_center_y = region.y() + region.height() / 2.0f; - RET_CHECK_GE(original_center_x, 0) << "Negative horizontal center."; - RET_CHECK_GE(original_center_y, 0) << "Negative vertical center."; - // Ensure that centered region of target size does not exceed frame size. - float center_x, center_y; - RET_CHECK(MathUtil::Clamp(half_width, scene_frame_width - half_width, - original_center_x, ¢er_x)); - RET_CHECK(MathUtil::Clamp(half_height, scene_frame_height - half_height, - original_center_y, ¢er_y)); - key_frame_compact_info->set_center_x(center_x); - key_frame_compact_info->set_center_y(center_y); - scene_summary->set_key_frame_center_min_x( - std::min(scene_summary->key_frame_center_min_x(), center_x)); - scene_summary->set_key_frame_center_max_x( - std::max(scene_summary->key_frame_center_max_x(), center_x)); - scene_summary->set_key_frame_center_min_y( - std::min(scene_summary->key_frame_center_min_y(), center_y)); - scene_summary->set_key_frame_center_max_y( - std::max(scene_summary->key_frame_center_max_y(), center_y)); - - scene_summary->set_crop_window_width( - std::max(scene_summary->crop_window_width(), region.width())); - scene_summary->set_crop_window_height( - std::max(scene_summary->crop_window_height(), region.height())); - - const float score = result.region_score(); - RET_CHECK_GE(score, 0.0) << "Negative score."; - key_frame_compact_info->set_score(result.region_score()); - scene_summary->set_key_frame_min_score( - std::min(scene_summary->key_frame_min_score(), score)); - scene_summary->set_key_frame_max_score( - std::max(scene_summary->key_frame_max_score(), score)); - } - - scene_summary->set_has_salient_region(has_salient_region); - scene_summary->set_has_required_salient_region(required_crop_region_union != - nullptr); - if (required_crop_region_union) { - *(scene_summary->mutable_key_frame_required_crop_region_union()) = - *required_crop_region_union; - } - const float success_rate = - static_cast(num_success_frames) / num_key_frames; - scene_summary->set_frame_success_rate(success_rate); - const float motion_x = - static_cast(scene_summary->key_frame_center_max_x() - - scene_summary->key_frame_center_min_x()) / - scene_frame_width; - scene_summary->set_horizontal_motion_amount(motion_x); - const float motion_y = - static_cast(scene_summary->key_frame_center_max_y() - - scene_summary->key_frame_center_min_y()) / - scene_frame_height; - scene_summary->set_vertical_motion_amount(motion_y); - return absl::OkStatus(); -} - -absl::Status ComputeSceneStaticBordersSize( - const std::vector& static_features, int* top_border_size, - int* bottom_border_size) { - RET_CHECK(top_border_size) << "Output top border size is null."; - RET_CHECK(bottom_border_size) << "Output bottom border size is null."; - - *top_border_size = -1; - for (int i = 0; i < static_features.size(); ++i) { - bool has_static_top_border = false; - for (const auto& feature : static_features[i].border()) { - if (feature.relative_position() == Border::TOP) { - has_static_top_border = true; - const int static_size = feature.border_position().height(); - *top_border_size = (*top_border_size > 0) - ? std::min(*top_border_size, static_size) - : static_size; - } - } - if (!has_static_top_border) { - *top_border_size = 0; - break; - } - } - - *bottom_border_size = -1; - for (int i = 0; i < static_features.size(); ++i) { - bool has_static_bottom_border = false; - for (const auto& feature : static_features[i].border()) { - if (feature.relative_position() == Border::BOTTOM) { - has_static_bottom_border = true; - const int static_size = feature.border_position().height(); - *bottom_border_size = (*bottom_border_size > 0) - ? std::min(*bottom_border_size, static_size) - : static_size; - } - } - if (!has_static_bottom_border) { - *bottom_border_size = 0; - break; - } - } - - *top_border_size = std::max(0, *top_border_size); - *bottom_border_size = std::max(0, *bottom_border_size); - return absl::OkStatus(); -} - -absl::Status FindSolidBackgroundColor( - const std::vector& static_features, - const std::vector& static_features_timestamps, - const double min_fraction_solid_background_color, - bool* has_solid_background, - PiecewiseLinearFunction* background_color_l_function, - PiecewiseLinearFunction* background_color_a_function, - PiecewiseLinearFunction* background_color_b_function) { - RET_CHECK(has_solid_background) << "Output boolean is null."; - RET_CHECK(background_color_l_function) << "Output color l function is null."; - RET_CHECK(background_color_a_function) << "Output color a function is null."; - RET_CHECK(background_color_b_function) << "Output color b function is null."; - - *has_solid_background = false; - int solid_background_frames = 0; - for (int i = 0; i < static_features.size(); ++i) { - if (static_features[i].has_solid_background()) { - solid_background_frames++; - const auto& color = static_features[i].solid_background(); - const int64 timestamp = static_features_timestamps[i]; - // BorderDetectionCalculator sets color assuming the input frame is - // BGR, but in reality we have RGB, so we need to revert it here. - // TODO remove this custom logic in BorderDetectionCalculator, - // original CroppingCalculator, and this calculator. - cv::Mat3f rgb_mat(1, 1, cv::Vec3b(color.b(), color.g(), color.r())); - // Necessary scaling of the RGB values from [0, 255] to [0, 1] based on: - // https://docs.opencv.org/2.4/modules/imgproc/doc/miscellaneous_transformations.html#cvtcolor - rgb_mat *= 1.0 / 255; - cv::Mat3f lab_mat(1, 1); - cv::cvtColor(rgb_mat, lab_mat, cv::COLOR_RGB2Lab); - // TODO change to piecewise constant interpolation if there is - // visual artifact. We can simply add one more point right before the - // next point with same value to mimic piecewise constant behavior. - const auto lab = lab_mat.at(0, 0); - background_color_l_function->AddPoint(timestamp, lab[0]); - background_color_a_function->AddPoint(timestamp, lab[1]); - background_color_b_function->AddPoint(timestamp, lab[2]); - } - } - - if (!static_features.empty() && - static_cast(solid_background_frames) / static_features.size() >= - min_fraction_solid_background_color) { - *has_solid_background = true; - } - return absl::OkStatus(); -} - -absl::Status AffineRetarget(const cv::Size& output_size, - const std::vector& frames, - const std::vector& affine_projection, - std::vector* cropped_frames) { - RET_CHECK(frames.size() == affine_projection.size()) - << "number of frames and retarget offsets must be the same."; - RET_CHECK(cropped_frames->size() == frames.size()) - << "Output vector cropped_frames must be populated with output images of " - "the same type, size and count."; - for (int i = 0; i < frames.size(); i++) { - RET_CHECK(frames[i].type() == (*cropped_frames)[i].type()) - << "input and output images must be the same type."; - const auto affine = affine_projection[i]; - RET_CHECK(affine.cols == 3) << "Affine matrix must be 2x3"; - RET_CHECK(affine.rows == 2) << "Affine matrix must be 2x3"; - cv::warpAffine(frames[i], (*cropped_frames)[i], affine, output_size); - } - return absl::OkStatus(); -} -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/utils.h b/mediapipe/examples/desktop/autoflip/quality/utils.h deleted file mode 100644 index 7285e8265..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/utils.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_ - -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales -// features back to the original frame size if features have been detected on a -// different frame size. -absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms, - const DetectionSet& detections, - const int original_frame_width, - const int original_frame_height, - const int feature_frame_width, - const int feature_frame_height, - KeyFrameInfo* key_frame_info); - -// Sorts required and non-required salient regions given a detection set. -absl::Status SortDetections(const DetectionSet& detections, - std::vector* required_regions, - std::vector* non_required_regions); - -// Sets the target crop size in KeyFrameCropOptions based on frame size and -// target aspect ratio so that the target crop size covers the biggest area -// possible in the frame. -absl::Status SetKeyFrameCropTarget(const int frame_width, - const int frame_height, - const double target_aspect_ratio, - KeyFrameCropOptions* crop_options); - -// Aggregates information from KeyFrameInfos and KeyFrameCropResults into -// SceneKeyFrameCropSummary. -absl::Status AggregateKeyFrameResults( - const KeyFrameCropOptions& key_frame_crop_options, - const std::vector& key_frame_crop_results, - const int scene_frame_width, const int scene_frame_height, - SceneKeyFrameCropSummary* scene_summary); - -// Computes the static top and border size across a scene given a vector of -// StaticFeatures over frames. -absl::Status ComputeSceneStaticBordersSize( - const std::vector& static_features, int* top_border_size, - int* bottom_border_size); - -// Finds the solid background colors in a scene from input StaticFeatures. -// Sets has_solid_background to true if the number of frames with solid -// background color exceeds given threshold, i.e., -// min_fraction_solid_background_color. Builds the background color -// interpolation functions in Lab space using input timestamps. -absl::Status FindSolidBackgroundColor( - const std::vector& static_features, - const std::vector& static_features_timestamps, - const double min_fraction_solid_background_color, - bool* has_solid_background, - PiecewiseLinearFunction* background_color_l_function, - PiecewiseLinearFunction* background_color_a_function, - PiecewiseLinearFunction* background_color_b_function); - -// Helpers to scale, clamp, and take union of rectangles. These functions do not -// check for pointers not being null or rectangles being valid. - -// Scales a rectangle given horizontal and vertical scaling factors. -template -void ScaleRect(const T& original_location, const double scale_x, - const double scale_y, Rect* scaled_location); - -// Converts a normalized rectangle to a rectangle given width and height. -void NormalizedRectToRect(const RectF& normalized_location, const int width, - const int height, Rect* location); - -// Clamps a rectangle to lie within [x0, y0] and [x1, y1]. Returns true if the -// rectangle has any overlapping with the target window. -absl::Status ClampRect(const int x0, const int y0, const int x1, const int y1, - Rect* location); - -// Convenience function to clamp a rectangle to lie within [0, 0] and -// [width, height]. -absl::Status ClampRect(const int width, const int height, Rect* location); - -// Enlarges a given rectangle to cover a new rectangle to be added. -void RectUnion(const Rect& rect_to_add, Rect* rect); - -// Performs an affine retarget on a list of input images. Output vector -// cropped_frames must be filled with Mats of the same size as output_size and -// type. -absl::Status AffineRetarget(const cv::Size& output_size, - const std::vector& frames, - const std::vector& affine_projection, - std::vector* cropped_frames); - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UTILS_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/utils_test.cc b/mediapipe/examples/desktop/autoflip/quality/utils_test.cc deleted file mode 100644 index b10e37855..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/utils_test.cc +++ /dev/null @@ -1,1039 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/utils.h" - -#include -#include - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/cropping.pb.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -using ::testing::HasSubstr; - -const int64 kTimestamp = 0; -const int kOriginalWidth = 100; -const int kOriginalHeight = 100; -const double kTargetAspectRatio = 1.5; -const int kNumKeyFrames = 5; -const int64 kKeyFrameTimestampDiff = 1e6 / kNumKeyFrames; -const int kTargetWidth = 50; -const int kTargetHeight = 50; - -// Makes a rectangle given the corner (x, y) and the size (width, height). -Rect MakeRect(const int x, const int y, const int width, const int height) { - Rect rect; - rect.set_x(x); - rect.set_y(y); - rect.set_width(width); - rect.set_height(height); - return rect; -} - -// Makes a rectangle given the center (center_x, center_y) and the half size -// (half_width, half_height). -Rect MakeRectFromCenterAndHalfSize(const float center_x, const float center_y, - const float half_width, - const float half_height) { - Rect rect; - rect.set_x(center_x - half_width); - rect.set_y(center_y - half_height); - rect.set_width(half_width * 2); - rect.set_height(half_height * 2); - return rect; -} - -// Computes the center of a rectangle as a pair (center_x, center_y). -std::pair RectCenter(const Rect& rect) { - return std::make_pair(rect.x() + rect.width() / 2.0, - rect.y() + rect.height() / 2.0); -} - -// Makes a normalized rectangle given the corner (x, y) and the size (width, -// height). -RectF MakeRectF(const double x, const double y, const double width, - const double height) { - RectF rectf; - rectf.set_x(x); - rectf.set_y(y); - rectf.set_width(width); - rectf.set_height(height); - return rectf; -} - -// Adds a detection to the detection set given location. -void AddDetectionFromLocation(const Rect& loc, DetectionSet* detections) { - auto* detection = detections->add_detections(); - *(detection->mutable_location()) = loc; -} - -// Adds a detection to the detection set given normalized location. -void AddDetectionFromNormalizedLocation(const RectF& normalized_loc, - DetectionSet* detections) { - auto* detection = detections->add_detections(); - *(detection->mutable_location_normalized()) = normalized_loc; -} - -// Checks whether two rectangles are the same, with an optional scaling factor. -bool CheckRectsEqual(const Rect& rect1, const Rect& rect2, - const double scale_x = 1.0, const double scale_y = 1.0) { - return (static_cast(round(scale_x * rect2.x())) == rect1.x() && - static_cast(round(scale_y * rect2.y())) == rect1.y() && - static_cast(round(scale_x * rect2.width())) == rect1.width() && - static_cast(round(scale_y * rect2.height())) == rect1.height()); -} - -// Adds a detection to the detection set given its score and whether it is -// required. -void AddDetectionFromScoreAndIsRequired(const double score, - const bool is_required, - DetectionSet* detections) { - auto* detection = detections->add_detections(); - detection->set_score(score); - detection->set_is_required(is_required); -} - -// Returns default settings for KeyFrameCropOptions. Populates target size to be -// the default target size. -KeyFrameCropOptions GetDefaultKeyFrameCropOptions() { - KeyFrameCropOptions key_frame_crop_options; - key_frame_crop_options.set_target_width(kTargetWidth); - key_frame_crop_options.set_target_height(kTargetHeight); - return key_frame_crop_options; -} - -// Returns default values for KeyFrameCropResults. Sets each frame to have -// covered all the required regions and non-required regions, and have required -// crop region (10, 10+20) x (10, 10+20), (full) crop region (0, 50) x (0, 50), -// and region score 1.0. -std::vector GetDefaultKeyFrameCropResults() { - std::vector key_frame_crop_results(kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - key_frame_crop_results[i].set_are_required_regions_covered_in_target_size( - true); - key_frame_crop_results[i].set_fraction_non_required_covered(1.0); - key_frame_crop_results[i].set_region_is_empty(false); - key_frame_crop_results[i].set_required_region_is_empty(false); - *(key_frame_crop_results[i].mutable_region()) = MakeRect(0, 0, 50, 50); - *(key_frame_crop_results[i].mutable_required_region()) = - MakeRect(10, 10, 20, 20); - key_frame_crop_results[i].set_region_score(1.0); - key_frame_crop_results[i].set_timestamp_ms(kKeyFrameTimestampDiff * i); - } - return key_frame_crop_results; -} - -// Checks that ScaleRect properly scales Rect and RectF objects. -TEST(UtilTest, ScaleRect) { - Rect scaled_rect; - ScaleRect(MakeRect(10, 10, 20, 30), 1.5, 2.0, &scaled_rect); - EXPECT_TRUE(CheckRectsEqual(scaled_rect, MakeRect(15, 20, 30, 60))); - - ScaleRect(MakeRectF(0.5, 0.9, 1.36, 0.748), 100, 50, &scaled_rect); - EXPECT_TRUE(CheckRectsEqual(scaled_rect, MakeRect(50, 45, 136, 37))); -} - -// Checks that NormalizedRectToRect properly converts a RectF object to a Rect -// object given width and height. -TEST(UtilTest, NormalizedRectToRect) { - const RectF normalized_rect = MakeRectF(0.1, 1.0, 2.5, 0.9); - Rect rect; - NormalizedRectToRect(normalized_rect, 100, 100, &rect); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(10, 100, 250, 90))); -} - -// Checks that ClampRect properly clamps a Rect object in [x0, y0] and [x1, y1]. -TEST(UtilTest, ClampRect) { - // Overlaps at a corner. - Rect rect = MakeRect(-10, -10, 80, 20); - MP_EXPECT_OK(ClampRect(0, 0, 100, 100, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(0, 0, 70, 10))); - // Overlaps on a side. - rect = MakeRect(10, -10, 80, 20); - MP_EXPECT_OK(ClampRect(0, 0, 100, 100, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(10, 0, 80, 10))); - // Inside. - rect = MakeRect(10, 10, 80, 10); - MP_EXPECT_OK(ClampRect(0, 0, 100, 100, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(10, 10, 80, 10))); - // Outside. - rect = MakeRect(-10, 10, 0, 0); - EXPECT_FALSE(ClampRect(0, 0, 100, 100, &rect).ok()); -} - -// Checks that ClampRect properly clamps a Rect object in [0, 0] and [width, -// height]. -TEST(UtilTest, ClampRectConvenienceFunction) { - Rect rect = MakeRect(-10, 0, 80, 10); - MP_EXPECT_OK(ClampRect(kOriginalWidth, kOriginalHeight, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(0, 0, 70, 10))); - rect = MakeRect(-10, 0, 120, 10); - MP_EXPECT_OK(ClampRect(kOriginalWidth, kOriginalHeight, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(0, 0, 100, 10))); - rect = MakeRect(10, 0, 70, 120); - MP_EXPECT_OK(ClampRect(kOriginalWidth, kOriginalHeight, &rect)); - EXPECT_TRUE(CheckRectsEqual(rect, MakeRect(10, 0, 70, 100))); -} - -// Checks that RectUnion properly takes the union of two Rect objects. -TEST(UtilTest, RectUnion) { - // Base rectangle and new rectangle are partially overlapping. - Rect base_rect = MakeRect(40, 40, 40, 40); - RectUnion(MakeRect(20, 30, 40, 40), &base_rect); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(20, 30, 60, 50))); - // Base rectangle contains new rectangle. - base_rect = MakeRect(40, 40, 40, 40); - RectUnion(MakeRect(50, 50, 10, 10), &base_rect); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(40, 40, 40, 40))); - // Base rectangle is contained by new rectangle. - base_rect = MakeRect(40, 40, 40, 40); - RectUnion(MakeRect(30, 30, 50, 50), &base_rect); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(30, 30, 50, 50))); - // Base rectangle and new rectangle are disjoint. - base_rect = MakeRect(40, 40, 40, 40); - RectUnion(MakeRect(15, 25, 20, 5), &base_rect); - EXPECT_TRUE(CheckRectsEqual(base_rect, MakeRect(15, 25, 65, 55))); -} - -// Checks that PackCropFrame fails on nullptr return object. -TEST(UtilTest, PackKeyFrameInfoFailsOnNullObject) { - DetectionSet detections; - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo* key_frame_info_ptr = nullptr; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, key_frame_info_ptr); - EXPECT_FALSE(status.ok()); -} - -// Checks that PackCropFrame fails on invalid frame size. -TEST(UtilTest, PackKeyFrameInfoFailsOnInvalidFrameSize) { - DetectionSet detections; - const int feature_width = -1, feature_height = 0; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info) - .ToString(); - EXPECT_THAT(status, testing::HasSubstr("Invalid frame size")); -} - -// Checks that PackCropFrame correctly packs timestamp. -TEST(UtilTest, PackKeyFrameInfoPacksTimestamp) { - DetectionSet detections; - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - EXPECT_EQ(key_frame_info.timestamp_ms(), kTimestamp); -} - -// Checks that PackCropFrame correctly packs detections. -TEST(UtilTest, PackKeyFrameInfoPacksDetections) { - DetectionSet detections; - AddDetectionFromLocation(MakeRect(0, 0, 10, 10), &detections); - AddDetectionFromLocation(MakeRect(20, 20, 30, 10), &detections); - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - EXPECT_EQ(key_frame_info.detections().detections_size(), - detections.detections_size()); - for (int i = 0; i < detections.detections_size(); ++i) { - const auto& original_rect = detections.detections(i).location(); - const auto& rect = key_frame_info.detections().detections(i).location(); - EXPECT_TRUE(CheckRectsEqual(rect, original_rect)); - } -} - -// Checks that PackCropFrame correctly converts normalized location to location. -TEST(UtilTest, PackKeyFrameInfoUnnormalizesLocations) { - DetectionSet detections; - AddDetectionFromNormalizedLocation(MakeRectF(0.1, 0.1, 0.1, 0.1), - &detections); - AddDetectionFromNormalizedLocation(MakeRectF(0.242, 0.256, 0.378, 0.399), - &detections); - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - const auto& out_rect1 = key_frame_info.detections().detections(0).location(); - const auto& out_rect2 = key_frame_info.detections().detections(1).location(); - EXPECT_TRUE(CheckRectsEqual(out_rect1, MakeRect(10, 10, 10, 10))); - EXPECT_TRUE(CheckRectsEqual(out_rect2, MakeRect(24, 26, 38, 40))); -} - -// Checks that PackCropFrame correctly scales location. -TEST(UtilTest, PackKeyFrameInfoScalesLocations) { - DetectionSet detections; - AddDetectionFromLocation(MakeRect(10, 10, 10, 10), &detections); - AddDetectionFromLocation(MakeRect(20, 20, 30, 30), &detections); - const double scaling = 2.0; - const int feature_width = kOriginalWidth / scaling; - const int feature_height = kOriginalHeight / scaling; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - EXPECT_EQ(key_frame_info.detections().detections_size(), - detections.detections_size()); - for (int i = 0; i < detections.detections_size(); ++i) { - const auto& original_rect = detections.detections(i).location(); - const auto& rect = key_frame_info.detections().detections(i).location(); - EXPECT_TRUE(CheckRectsEqual(rect, original_rect, scaling, scaling)); - } -} - -// Checks that PackCropFrame correctly clamps location to be within frame size. -TEST(UtilTest, PackKeyFrameInfoClampsLocations) { - DetectionSet detections; - AddDetectionFromLocation(MakeRect(10, 10, 100, 10), &detections); - AddDetectionFromLocation(MakeRect(0, -10, 110, 100), &detections); - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - EXPECT_EQ(key_frame_info.detections().detections_size(), - detections.detections_size()); - const auto& out_rect1 = key_frame_info.detections().detections(0).location(); - const auto& out_rect2 = key_frame_info.detections().detections(1).location(); - EXPECT_TRUE(CheckRectsEqual(out_rect1, MakeRect(10, 10, 90, 10))); - EXPECT_TRUE(CheckRectsEqual(out_rect2, MakeRect(0, 0, 100, 90))); -} - -// Checks that PackCropFrame correctly clamps normalized location to be within -// frame size. -TEST(UtilTest, PackKeyFrameInfoClampsNormalizedLocations) { - DetectionSet detections; - AddDetectionFromNormalizedLocation(MakeRectF(-0.05, 0.3, 0.4, 0.8), - &detections); - AddDetectionFromNormalizedLocation(MakeRectF(0.05, -0.1, 1.0, 1.1), - &detections); - const int feature_width = kOriginalWidth, feature_height = kOriginalHeight; - - KeyFrameInfo key_frame_info; - const auto status = - PackKeyFrameInfo(kTimestamp, detections, kOriginalWidth, kOriginalHeight, - feature_width, feature_height, &key_frame_info); - - MP_EXPECT_OK(status); - EXPECT_EQ(key_frame_info.detections().detections_size(), - detections.detections_size()); - const auto& out_rect1 = key_frame_info.detections().detections(0).location(); - const auto& out_rect2 = key_frame_info.detections().detections(1).location(); - EXPECT_TRUE(CheckRectsEqual(out_rect1, MakeRect(0, 30, 35, 70))); - EXPECT_TRUE(CheckRectsEqual(out_rect2, MakeRect(5, 0, 95, 100))); -} - -// Checks that SortDetections correctly handles empty regions. -TEST(UtilTest, SortDetectionsHandlesEmptyRegions) { - DetectionSet detections; - std::vector required, non_required; - MP_EXPECT_OK(SortDetections(detections, &required, &non_required)); - EXPECT_EQ(detections.detections_size(), - required.size() + non_required.size()); -} - -// Checks that SortDetections correctly separates required and non-required -// salient regions. -TEST(UtilTest, SortDetectionsSeparatesRequiredAndNonRequiredRegions) { - DetectionSet detections; - auto gen_bool = std::bind(std::uniform_int_distribution<>(0, 1), - std::default_random_engine()); - for (int i = 0; i < 100; ++i) { - const bool is_required = gen_bool(); - AddDetectionFromScoreAndIsRequired(1.0, is_required, &detections); - } - - std::vector required, non_required; - MP_EXPECT_OK(SortDetections(detections, &required, &non_required)); - EXPECT_EQ(detections.detections_size(), - required.size() + non_required.size()); - for (int i = 0; i < required.size(); ++i) { - EXPECT_TRUE(required[i].is_required()); - } - for (int i = 0; i < non_required.size(); ++i) { - EXPECT_FALSE(non_required[i].is_required()); - } -} - -// Checks that SortDetections correctly sorts regions based on scores. -TEST(UtilTest, SortDetectionsSortsRegions) { - DetectionSet detections; - auto gen_score = std::bind(std::uniform_real_distribution<>(0.0, 1.0), - std::default_random_engine()); - auto gen_bool = std::bind(std::uniform_int_distribution<>(0, 1), - std::default_random_engine()); - for (int i = 0; i < 100; ++i) { - const double score = gen_score(); - const bool is_required = gen_bool(); - AddDetectionFromScoreAndIsRequired(score, is_required, &detections); - } - - std::vector required, non_required; - MP_EXPECT_OK(SortDetections(detections, &required, &non_required)); - EXPECT_EQ(detections.detections_size(), - required.size() + non_required.size()); - for (int i = 0; i < required.size() - 1; ++i) { - EXPECT_GE(required[i].score(), required[i + 1].score()); - } - for (int i = 0; i < non_required.size() - 1; ++i) { - EXPECT_GE(non_required[i].score(), non_required[i + 1].score()); - } -} - -// Checks that SetKeyFrameCropTarget checks KeyFrameCropOptions is not null. -TEST(UtilTest, SetKeyFrameCropTargetChecksKeyFrameCropOptionsNotNull) { - const auto status = SetKeyFrameCropTarget(kOriginalWidth, kOriginalHeight, - kTargetAspectRatio, nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("KeyFrameCropOptions is null.")); -} - -// Checks that SetKeyFrameCropTarget checks frame size and target aspect ratio -// are valid. -TEST(UtilTest, SetKeyFrameCropTargetChecksFrameSizeAndTargetAspectRatioValid) { - KeyFrameCropOptions crop_options; - auto status = SetKeyFrameCropTarget(0, kOriginalHeight, kTargetAspectRatio, - &crop_options); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("Frame width is non-positive.")); - - status = - SetKeyFrameCropTarget(kOriginalWidth, kOriginalHeight, 0, &crop_options); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("Target aspect ratio is non-positive.")); -} - -// Checks that SetKeyFrameCropTarget correctly sets the target crop size. -TEST(UtilTest, SetKeyFrameCropTargetSetsTargetSizeCorrectly) { - KeyFrameCropOptions crop_options; - // 1a) square -> square. - MP_EXPECT_OK(SetKeyFrameCropTarget(101, 101, 1.0, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 101); - EXPECT_EQ(crop_options.target_height(), 101); - // 1b) square -> landscape. - MP_EXPECT_OK(SetKeyFrameCropTarget(101, 101, 1.5, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 101); - EXPECT_EQ(crop_options.target_height(), 67); - // 1c) square -> vertical. - MP_EXPECT_OK(SetKeyFrameCropTarget(101, 101, 0.5, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 51); - EXPECT_EQ(crop_options.target_height(), 101); - // 2a) landscape -> square. - MP_EXPECT_OK(SetKeyFrameCropTarget(128, 72, 1.0, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 72); - EXPECT_EQ(crop_options.target_height(), 72); - // 2b) landscape -> more landscape. - MP_EXPECT_OK(SetKeyFrameCropTarget(128, 72, 2.0, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 128); - EXPECT_EQ(crop_options.target_height(), 64); - // 2c) landscape -> vertical. - MP_EXPECT_OK(SetKeyFrameCropTarget(128, 72, 0.7, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 50); - EXPECT_EQ(crop_options.target_height(), 72); - // 3a) vertical -> square. - MP_EXPECT_OK(SetKeyFrameCropTarget(90, 160, 1.0, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 90); - EXPECT_EQ(crop_options.target_height(), 90); - // 3b) vertical -> more vertical. - MP_EXPECT_OK(SetKeyFrameCropTarget(90, 160, 0.36, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 58); - EXPECT_EQ(crop_options.target_height(), 160); - // 3c) vertical -> landscape. - MP_EXPECT_OK(SetKeyFrameCropTarget(90, 160, 1.2, &crop_options)); - EXPECT_EQ(crop_options.target_width(), 90); - EXPECT_EQ(crop_options.target_height(), 75); -} - -// Checks that AggregateKeyFrameResults checks output pointer is not null. -TEST(UtilTest, AggregateKeyFrameResultsChecksOutputNotNull) { - const auto status = AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kOriginalWidth, kOriginalHeight, nullptr); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Output SceneKeyFrameCropSummary is null.")); -} - -// Checks that AggregateKeyFrameResults handles the case of no key frames. -TEST(UtilTest, AggregateKeyFrameResultsHandlesNoKeyFrames) { - std::vector key_frame_crop_results(0); - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); -} - -// Checks that AggregateKeyFrameResults checks that frame size is valid. -TEST(UtilTest, AggregateKeyFrameResultsChecksFrameSizeValid) { - SceneKeyFrameCropSummary scene_summary; - const auto status = AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kOriginalWidth, 0, &scene_summary); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Non-positive frame height.")); -} - -// Checks that AggregateKeyFrameResults checks that target size is valid. -TEST(UtilTest, AggregateKeyFrameResultsChecksTargetSizeValid) { - KeyFrameCropOptions key_frame_crop_options; - key_frame_crop_options.set_target_width(0); - SceneKeyFrameCropSummary scene_summary; - - const auto status = AggregateKeyFrameResults( - key_frame_crop_options, GetDefaultKeyFrameCropResults(), kOriginalWidth, - kOriginalHeight, &scene_summary); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Non-positive target width.")); -} - -// Checks that AggregateKeyFrameResults checks that target size does not exceed -// frame size. -TEST(UtilTest, AggregateKeyFrameResultsChecksTargetSizeNotExceedFrameSize) { - auto key_frame_crop_options = GetDefaultKeyFrameCropOptions(); - key_frame_crop_options.set_target_width(kOriginalWidth + 1); - SceneKeyFrameCropSummary scene_summary; - - const auto status = AggregateKeyFrameResults( - key_frame_crop_options, GetDefaultKeyFrameCropResults(), kOriginalWidth, - kOriginalHeight, &scene_summary); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - HasSubstr("Target width exceeds frame width.")); -} - -// Checks that AggregateKeyFrameResults packs KeyFrameCompactInfos. -TEST(UtilTest, AggregateKeyFrameResultsPacksKeyFrameCompactInfos) { - const auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - - EXPECT_EQ(scene_summary.num_key_frames(), kNumKeyFrames); - EXPECT_EQ(scene_summary.key_frame_compact_infos_size(), kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - const auto& compact_info = scene_summary.key_frame_compact_infos(i); - EXPECT_EQ(compact_info.timestamp_ms(), - key_frame_crop_results[i].timestamp_ms()); - const auto center = RectCenter(key_frame_crop_results[i].region()); - EXPECT_FLOAT_EQ(compact_info.center_x(), center.first); - EXPECT_FLOAT_EQ(compact_info.center_y(), center.second); - EXPECT_FLOAT_EQ(compact_info.score(), - key_frame_crop_results[i].region_score()); - } -} - -// Checks that AggregateKeyFrameResults ensures the centered region of target -// size fits in frame bound. -TEST(UtilTest, AggregateKeyFrameResultsEnsuresCropRegionFitsInFrame) { - std::vector key_frame_crop_results(1); - auto* crop_region = key_frame_crop_results[0].mutable_region(); - crop_region->set_x(0); - crop_region->set_y(0); - crop_region->set_width(10); - crop_region->set_height(10); - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - - EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); - const auto& compact_info = scene_summary.key_frame_compact_infos(0); - const float left = compact_info.center_x() - kTargetWidth / 2.0f; - const float right = compact_info.center_x() + kTargetWidth / 2.0f; - const float top = compact_info.center_y() - kTargetWidth / 2.0f; - const float bottom = compact_info.center_y() + kTargetWidth / 2.0f; - // Crop window is in the frame. - EXPECT_GE(left, 0); - EXPECT_LE(right, kOriginalWidth); - EXPECT_GE(top, 0); - EXPECT_LE(bottom, kOriginalHeight); - // Crop window covers input crop region. - EXPECT_LE(left, crop_region->x()); - EXPECT_GE(right, crop_region->x() + crop_region->width()); - EXPECT_LE(top, crop_region->y()); - EXPECT_GE(bottom, crop_region->y() + crop_region->height()); -} - -// Checks that AggregateKeyFrameResults sets centers and scores to -1.0 for key -// frames with empty regions. -TEST(UtilTest, - AggregateKeyFrameResultsSetsMinusOneForKeyFramesWithEmptyRegions) { - std::vector key_frame_crop_results(1); - key_frame_crop_results[0].set_region_is_empty(true); - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - - const auto& compact_info = scene_summary.key_frame_compact_infos(0); - EXPECT_FLOAT_EQ(compact_info.center_x(), -1.0f); - EXPECT_FLOAT_EQ(compact_info.center_y(), -1.0f); - EXPECT_FLOAT_EQ(compact_info.score(), -1.0f); -} - -// Checks that AggregateKeyFrameResults rejects negative center. -TEST(UtilTest, AggregateKeyFrameResultsRejectsNegativeCenter) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - auto* region = key_frame_crop_results[0].mutable_region(); - *region = MakeRectFromCenterAndHalfSize(10, -1.0, 10, 10); - SceneKeyFrameCropSummary scene_summary; - - const auto status = AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Negative vertical center.")); -} - -// Checks that AggregateKeyFrameResults rejects negative score. -TEST(UtilTest, AggregateKeyFrameResultsRejectsNegativeScore) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - key_frame_crop_results[0].set_region_score(-1.0); - SceneKeyFrameCropSummary scene_summary; - - const auto status = AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), HasSubstr("Negative score.")); -} - -// Checks that AggregateKeyFrameResults properly sets center ranges. -TEST(UtilTest, AggregateKeyFrameResultsSetsCenterRanges) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - const std::vector centers_x = {30.0, 20.0, 45.0, 3.0, 10.0}; - const std::vector centers_y = {10.0, 0.0, 5.0, 30.0, 20.0}; - const int half_width = 25, half_height = 25; - for (int i = 0; i < kNumKeyFrames; ++i) { - auto* region = key_frame_crop_results[i].mutable_region(); - *region = MakeRectFromCenterAndHalfSize(centers_x[i], centers_y[i], - half_width, half_height); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - - EXPECT_FLOAT_EQ(scene_summary.key_frame_center_min_x(), 25.0f); - EXPECT_FLOAT_EQ(scene_summary.key_frame_center_max_x(), 45.0f); - EXPECT_FLOAT_EQ(scene_summary.key_frame_center_min_y(), 25.0f); - EXPECT_FLOAT_EQ(scene_summary.key_frame_center_max_y(), 30.0f); -} - -// Checks that AggregateKeyFrameResults properly sets score range. -TEST(UtilTest, AggregateKeyFrameResultsSetsScoreRange) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - const std::vector scores = {0.9, 0.1, 1.2, 2.0, 0.5}; - for (int i = 0; i < kNumKeyFrames; ++i) { - key_frame_crop_results[i].set_region_score(scores[i]); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - - EXPECT_FLOAT_EQ(scene_summary.key_frame_min_score(), - *std::min_element(scores.begin(), scores.end())); - EXPECT_FLOAT_EQ(scene_summary.key_frame_max_score(), - *std::max_element(scores.begin(), scores.end())); -} - -// Checks that AggregateKeyFrameResults sets crop window size to target size -// when the crop regions fit in target size. -TEST(UtilTest, AggregateKeyFrameResultsSetsCropWindowSizeToTargetSize) { - SceneKeyFrameCropSummary scene_summary; - MP_EXPECT_OK(AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kOriginalWidth, kOriginalHeight, &scene_summary)); - EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); -} - -// Checks that AggregateKeyFrameResults properly sets crop window size when the -// crop regions exceed target size. -TEST(UtilTest, AggregateKeyFrameResultsSetsCropWindowSizeExceedingTargetSize) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - key_frame_crop_results[0].mutable_region()->set_width(kTargetWidth + 1); - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - EXPECT_EQ(scene_summary.crop_window_width(), kTargetWidth + 1); - EXPECT_EQ(scene_summary.crop_window_height(), kTargetHeight); -} - -// Checks that AggregateKeyFrameResults sets has salient region to true when -// there are salient regions. -TEST(UtilTest, AggregateKeyFrameResultsSetsHasSalientRegionTrue) { - SceneKeyFrameCropSummary scene_summary; - MP_EXPECT_OK(AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kOriginalWidth, kOriginalHeight, &scene_summary)); - EXPECT_TRUE(scene_summary.has_salient_region()); -} - -// Checks that AggregateKeyFrameResults sets has salient region to false when -// there are no salient regions. -TEST(UtilTest, AggregateKeyFrameResultsSetsHasSalientRegionFalse) { - std::vector key_frame_crop_results(kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - key_frame_crop_results[i].set_region_is_empty(true); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - EXPECT_FALSE(scene_summary.has_salient_region()); -} - -// Checks that AggregateKeyFrameResults sets has required salient region to true -// when there are required salient regions. -TEST(UtilTest, AggregateKeyFrameResultsSetsHasRequiredSalientRegionTrue) { - SceneKeyFrameCropSummary scene_summary; - MP_EXPECT_OK(AggregateKeyFrameResults( - GetDefaultKeyFrameCropOptions(), GetDefaultKeyFrameCropResults(), - kOriginalWidth, kOriginalHeight, &scene_summary)); - EXPECT_TRUE(scene_summary.has_required_salient_region()); -} - -// Checks that AggregateKeyFrameResults sets has required salient region to -// false when there are no required salient regions. -TEST(UtilTest, AggregateKeyFrameResultsSetsHasRequiredSalientRegionFalse) { - std::vector key_frame_crop_results(kNumKeyFrames); - for (int i = 0; i < kNumKeyFrames; ++i) { - key_frame_crop_results[i].set_required_region_is_empty(true); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - EXPECT_FALSE(scene_summary.has_required_salient_region()); -} - -// Checks that AggregateKeyFrameResults sets key frame required crop region -// union. -TEST(UtilTest, AggregateKeyFrameResultsSetsKeyFrameRequiredCropRegionUnion) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - for (int i = 0; i < kNumKeyFrames; ++i) { - *key_frame_crop_results[i].mutable_required_region() = - MakeRect(i, 0, 50, 50); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - const auto& required_crop_region_union = - scene_summary.key_frame_required_crop_region_union(); - EXPECT_EQ(required_crop_region_union.x(), 0); - EXPECT_EQ(required_crop_region_union.width(), 50 + kNumKeyFrames - 1); -} - -// Checks that AggregateKeyFrameResults properly sets frame success rate. -TEST(UtilTest, AggregateKeyFrameResultsSetsFrameSuccessRate) { - const int num_success_frames = 3; - const float success_rate = - static_cast(num_success_frames) / kNumKeyFrames; - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - for (int i = 0; i < kNumKeyFrames; ++i) { - const bool successful = i < num_success_frames ? true : false; - key_frame_crop_results[i].set_are_required_regions_covered_in_target_size( - successful); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - EXPECT_FLOAT_EQ(scene_summary.frame_success_rate(), success_rate); -} - -// Checks that AggregateKeyFrameResults properly sets motion. -TEST(UtilTest, AggregateKeyFrameResultsSetsMotion) { - auto key_frame_crop_results = GetDefaultKeyFrameCropResults(); - const std::vector centers_x = {30.0, 55.0, 45.0, 30.0, 60.0}; - const std::vector centers_y = {30.0, 40.0, 50.0, 45.0, 25.0}; - const float motion_x = (60.0 - 30.0) / kOriginalWidth; - const float motion_y = (50.0 - 25.0) / kOriginalHeight; - const int half_width = 25, half_height = 25; - for (int i = 0; i < kNumKeyFrames; ++i) { - auto* region = key_frame_crop_results[i].mutable_region(); - *region = MakeRectFromCenterAndHalfSize(centers_x[i], centers_y[i], - half_width, half_height); - } - SceneKeyFrameCropSummary scene_summary; - - MP_EXPECT_OK(AggregateKeyFrameResults(GetDefaultKeyFrameCropOptions(), - key_frame_crop_results, kOriginalWidth, - kOriginalHeight, &scene_summary)); - EXPECT_FLOAT_EQ(scene_summary.horizontal_motion_amount(), motion_x); - EXPECT_FLOAT_EQ(scene_summary.vertical_motion_amount(), motion_y); -} - -// Checks that ComputeSceneStaticBordersSize checks output not null. -TEST(UtilTest, ComputeSceneStaticBordersSizeChecksOutputNotNull) { - std::vector static_features; - int bottom_border_size = 0; - const auto status = ComputeSceneStaticBordersSize(static_features, nullptr, - &bottom_border_size); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("Output top border size is null.")); -} -// Checks that ComputeSceneStaticBordersSize returns 0 when there are no static -// borders. -TEST(UtilTest, ComputeSceneStaticBordersSizeNoBorder) { - std::vector static_features(10); - int top_border_size = 0, bottom_border_size = 0; - MP_EXPECT_OK(ComputeSceneStaticBordersSize(static_features, &top_border_size, - &bottom_border_size)); - EXPECT_EQ(top_border_size, 0); - EXPECT_EQ(bottom_border_size, 0); -} - -// Checks that ComputeSceneStaticBordersSize correctly computes static border -// size. -TEST(UtilTest, ComputeSceneStaticBordersSizeHasBorders) { - const int num_frames = 6; - const std::vector top_borders = {10, 9, 8, 9, 10, 5}; - const std::vector bottom_borders = {7, 7, 7, 7, 6, 7}; - std::vector static_features(num_frames); - for (int i = 0; i < num_frames; ++i) { - auto& features = static_features[i]; - auto* top_part = features.add_border(); - top_part->set_relative_position(Border::TOP); - top_part->mutable_border_position()->set_height(top_borders[i]); - auto* bottom_part = features.add_border(); - bottom_part->set_relative_position(Border::BOTTOM); - bottom_part->mutable_border_position()->set_height(bottom_borders[i]); - } - int top_border_size = 0, bottom_border_size = 0; - MP_EXPECT_OK(ComputeSceneStaticBordersSize(static_features, &top_border_size, - &bottom_border_size)); - EXPECT_EQ(top_border_size, 5); - EXPECT_EQ(bottom_border_size, 6); -} - -// Checks that FindSolidBackgroundColor checks output not null. -TEST(UtilTest, FindSolidBackgroundColorChecksOutputNotNull) { - std::vector static_features; - std::vector static_features_timestamps; - const double min_fraction_solid_background_color = 0.8; - bool has_solid_background_color; - PiecewiseLinearFunction l_function, a_function, b_function; - - auto status = - FindSolidBackgroundColor(static_features, static_features_timestamps, - min_fraction_solid_background_color, nullptr, - &l_function, &a_function, &b_function); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), testing::HasSubstr("Output boolean is null.")); - - status = FindSolidBackgroundColor(static_features, static_features_timestamps, - min_fraction_solid_background_color, - &has_solid_background_color, nullptr, - &a_function, &b_function); - EXPECT_FALSE(status.ok()); - EXPECT_THAT(status.ToString(), - testing::HasSubstr("Output color l function is null.")); -} - -// Checks that FindSolidBackgroundColor returns true when there is solid -// background color. -TEST(UtilTest, FindSolidBackgroundColorReturnsTrue) { - std::vector static_features(1); - auto* color = static_features[0].mutable_solid_background(); - color->set_r(255); - color->set_g(255); - color->set_b(255); - std::vector static_features_timestamps(1); - static_features_timestamps[0] = 0; - const double min_fraction_solid_background_color = 0.8; - bool has_solid_background_color; - PiecewiseLinearFunction l_function, a_function, b_function; - - MP_EXPECT_OK(FindSolidBackgroundColor( - static_features, static_features_timestamps, - min_fraction_solid_background_color, &has_solid_background_color, - &l_function, &a_function, &b_function)); - EXPECT_TRUE(has_solid_background_color); -} - -// Checks that FindSolidBackgroundColor returns false when there is no solid -// background color. -TEST(UtilTest, FindSolidBackgroundColorReturnsFalse) { - std::vector static_features(1); - std::vector static_features_timestamps(1); - static_features_timestamps[0] = 0; - const double min_fraction_solid_background_color = 0.8; - bool has_solid_background_color; - PiecewiseLinearFunction l_function, a_function, b_function; - - MP_EXPECT_OK(FindSolidBackgroundColor( - static_features, static_features_timestamps, - min_fraction_solid_background_color, &has_solid_background_color, - &l_function, &a_function, &b_function)); - EXPECT_FALSE(has_solid_background_color); -} - -// Checks that FindSolidBackgroundColor sets the interpolation functions. -TEST(UtilTest, FindSolidBackgroundColorSetsInterpolationFunctions) { - const uint8 rgb1[] = {255, 255, 0}; // cyan in bgr - const double lab1[] = {91.1133, -48.0938, -14.125}; - const int64 time1 = 0; - const uint8 rgb2[] = {255, 0, 255}; // magenta in bgr - const double lab2[] = {60.321, 98.2344, -60.8281}; - const int64 time2 = 2000; - std::vector static_features(2); - auto* color1 = static_features[0].mutable_solid_background(); - color1->set_r(rgb1[0]); - color1->set_g(rgb1[1]); - color1->set_b(rgb1[2]); - auto* color2 = static_features[1].mutable_solid_background(); - color2->set_r(rgb2[0]); - color2->set_g(rgb2[1]); - color2->set_b(rgb2[2]); - std::vector static_features_timestamps(2); - static_features_timestamps[0] = time1; - static_features_timestamps[1] = time2; - const double min_fraction_solid_background_color = 0.8; - bool has_solid_background_color; - PiecewiseLinearFunction l_function, a_function, b_function; - - MP_EXPECT_OK(FindSolidBackgroundColor( - static_features, static_features_timestamps, - min_fraction_solid_background_color, &has_solid_background_color, - &l_function, &a_function, &b_function)); - - EXPECT_TRUE(has_solid_background_color); - EXPECT_LE(std::fabs(l_function.Evaluate(time1) - lab1[0]), 1e-2f); - EXPECT_LE(std::fabs(a_function.Evaluate(time1) - lab1[1]), 1e-2f); - EXPECT_LE(std::fabs(b_function.Evaluate(time1) - lab1[2]), 1e-2f); - EXPECT_LE(std::fabs(l_function.Evaluate(time2) - lab2[0]), 1e-2f); - EXPECT_LE(std::fabs(a_function.Evaluate(time2) - lab2[1]), 1e-2f); - EXPECT_LE(std::fabs(b_function.Evaluate(time2) - lab2[2]), 1e-2f); - - EXPECT_LE(std::fabs(l_function.Evaluate((time1 + time2) / 2.0) - - (lab1[0] + lab2[0]) / 2.0), - 1e-2f); - EXPECT_LE(std::fabs(a_function.Evaluate((time1 + time2) / 2.0) - - (lab1[1] + lab2[1]) / 2.0), - 1e-2f); - EXPECT_LE(std::fabs(b_function.Evaluate((time1 + time2) / 2.0) - - (lab1[2] + lab2[2]) / 2.0), - 1e-2f); -} - -TEST(UtilTest, TestAffineRetargeterPass) { - std::vector transforms; - std::vector frames; - std::vector results; - for (int i = 0; i < 5; i++) { - cv::Mat transform = cv::Mat(2, 3, CV_32FC1); - transform.at(0, 0) = 1; - transform.at(0, 1) = 0; - transform.at(1, 0) = 0; - transform.at(1, 1) = 1; - transform.at(0, 2) = -i * 50; - transform.at(1, 2) = 0; - transforms.push_back(transform); - cv::Mat image = cv::Mat::zeros(1080, 1920, CV_8UC3); - cv::Vec3b val; - val[0] = 255; - val[1] = 255; - val[2] = 255; - image(cv::Rect(0, 0, 395, 1080)).setTo(val); - frames.push_back(image); - - cv::Mat image_out = cv::Mat::zeros(500, 1920, CV_8UC3); - results.push_back(image_out); - } - - MP_ASSERT_OK( - AffineRetarget(cv::Size(500, 1080), frames, transforms, &results)); - ASSERT_EQ(results.size(), 5); - for (int i = 0; i < 5; i++) { - EXPECT_GT(results[i].at(540, 390 - i * 50)[0], 250); - EXPECT_GT(results[i].at(540, 390 - i * 50)[1], 250); - EXPECT_GT(results[i].at(540, 390 - i * 50)[2], 250); - EXPECT_LT(results[i].at(540, 400 - i * 50)[0], 5); - EXPECT_LT(results[i].at(540, 400 - i * 50)[1], 5); - EXPECT_LT(results[i].at(540, 400 - i * 50)[2], 5); - } -} - -TEST(UtilTest, TestAffineRetargeterFail) { - std::vector transforms; - std::vector frames; - std::vector results; - - cv::Mat dummy; - transforms.push_back(dummy); - frames.push_back(dummy); - - EXPECT_THAT( - AffineRetarget(cv::Size(500, 1080), frames, transforms, &results) - .ToString(), - testing::HasSubstr("Output vector cropped_frames must be populated")); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc b/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc deleted file mode 100644 index 9ae612004..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h" - -#include - -#include -#include -#include -#include - -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_builder.h" - -// Weight threshold for computing a value. -constexpr float kEpsilon = 0.0001; - -namespace mediapipe { -namespace autoflip { -namespace { - -// Crop the given rectangle so that it fits in the given 2D matrix. -void CropRectToMat(const cv::Mat& image, cv::Rect* rect) { - int x = std::min(std::max(rect->x, 0), image.cols); - int y = std::min(std::max(rect->y, 0), image.rows); - int w = std::min(std::max(rect->x + rect->width, 0), image.cols) - x; - int h = std::min(std::max(rect->y + rect->height, 0), image.rows) - y; - *rect = cv::Rect(x, y, w, h); -} - -} // namespace - -VisualScorer::VisualScorer(const VisualScorerOptions& options) - : options_(options) {} - -absl::Status VisualScorer::CalculateScore(const cv::Mat& image, - const SalientRegion& region, - float* score) const { - const float weight_sum = options_.area_weight() + - options_.sharpness_weight() + - options_.colorfulness_weight(); - - // Crop the region to fit in the image. - cv::Rect region_rect; - if (region.has_location()) { - region_rect = - cv::Rect(region.location().x(), region.location().y(), - region.location().width(), region.location().height()); - } else if (region.has_location_normalized()) { - region_rect = cv::Rect(region.location_normalized().x() * image.cols, - region.location_normalized().y() * image.rows, - region.location_normalized().width() * image.cols, - region.location_normalized().height() * image.rows); - } else { - return mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) - << "Unset region location."; - } - - CropRectToMat(image, ®ion_rect); - if (region_rect.area() == 0) { - *score = 0; - return absl::OkStatus(); - } - - // Compute a score based on area covered by this region. - const float area_score = - options_.area_weight() * region_rect.area() / (image.cols * image.rows); - - // Convert the visible region to cv::Mat. - cv::Mat image_region_mat = image(region_rect); - - // Compute a score from sharpness. - - float sharpness_score_result = 0.0; - if (options_.sharpness_weight() > kEpsilon) { - // TODO: implement a sharpness score or remove this code block. - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "sharpness scorer is not yet implemented, please set weight to " - "0.0"; - } - const float sharpness_score = - options_.sharpness_weight() * sharpness_score_result; - - // Compute a colorfulness score. - float colorfulness_score = 0; - if (options_.colorfulness_weight() > kEpsilon) { - MP_RETURN_IF_ERROR( - CalculateColorfulness(image_region_mat, &colorfulness_score)); - colorfulness_score *= options_.colorfulness_weight(); - } - - *score = (area_score + sharpness_score + colorfulness_score) / weight_sum; - if (*score > 1.0f || *score < 0.0f) { - LOG(WARNING) << "Score of region outside expected range: " << *score; - } - return absl::OkStatus(); -} - -absl::Status VisualScorer::CalculateColorfulness(const cv::Mat& image, - float* colorfulness) const { - // Convert the image to HSV. - cv::Mat image_hsv; - cv::cvtColor(image, image_hsv, CV_RGB2HSV); - - // Mask out pixels that are too dark or too bright. - cv::Mat mask(image.rows, image.cols, CV_8UC1); - bool empty_mask = true; - for (int x = 0; x < image.cols; ++x) { - for (int y = 0; y < image.rows; ++y) { - const cv::Vec3b& pixel = image.at(x, y); - const bool is_usable = - (std::min(pixel.val[0], std::min(pixel.val[1], pixel.val[2])) < 250 && - std::max(pixel.val[0], std::max(pixel.val[1], pixel.val[2])) > 5); - mask.at(y, x) = is_usable ? 255 : 0; - if (is_usable) empty_mask = false; - } - } - - // If the mask is empty, return. - if (empty_mask) { - *colorfulness = 0; - return absl::OkStatus(); - } - - // Generate a 2D histogram (hue/saturation). - cv::MatND hs_histogram; - const int kHueBins = 10, kSaturationBins = 8; - const int kHistogramChannels[] = {0, 1}; - const int kHistogramBinNum[] = {kHueBins, kSaturationBins}; - const float kHueRange[] = {0, 180}; - const float kSaturationRange[] = {0, 256}; - const float* kHistogramRange[] = {kHueRange, kSaturationRange}; - cv::calcHist(&image_hsv, 1, kHistogramChannels, mask, hs_histogram, - 2 /* histogram dims */, kHistogramBinNum, kHistogramRange, - true /* uniform */, false /* accumulate */); - - // Convert to a hue histogram and weigh saturated pixels more. - std::vector hue_histogram(kHueBins, 0.0f); - float hue_sum = 0.0f; - for (int bin_s = 0; bin_s < kSaturationBins; ++bin_s) { - const float weight = std::pow(2.0f, bin_s); - for (int bin_h = 0; bin_h < kHueBins; ++bin_h) { - float value = hs_histogram.at(bin_h, bin_s) * weight; - hue_histogram[bin_h] += value; - hue_sum += value; - } - } - if (hue_sum == 0.0f) { - *colorfulness = 0; - return absl::OkStatus(); - } - - // Compute the histogram entropy. - *colorfulness = 0; - for (int bin = 0; bin < kHueBins; ++bin) { - float value = hue_histogram[bin] / hue_sum; - if (value > 0.0f) { - *colorfulness -= value * std::log(value); - } - } - *colorfulness /= std::log(2.0f); - - return absl::OkStatus(); -} - -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.h b/mediapipe/examples/desktop/autoflip/quality/visual_scorer.h deleted file mode 100644 index b2c6d3af7..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_ -#define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_ - -#include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" -#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { -namespace autoflip { - -// This class scores a SalientRegion within an image based on weighted averages -// of various signals computed on the patch. -class VisualScorer { - public: - explicit VisualScorer(const VisualScorerOptions& options); - - // Computes a score on a salientregion and returns a value [0...1]. - absl::Status CalculateScore(const cv::Mat& image, const SalientRegion& region, - float* score) const; - - private: - absl::Status CalculateColorfulness(const cv::Mat& image, - float* colorfulness) const; - - VisualScorerOptions options_; -}; - -} // namespace autoflip -} // namespace mediapipe - -#endif // MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_VISUAL_SCORER_H_ diff --git a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto b/mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto deleted file mode 100644 index 4623e1e02..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.proto +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; - -package mediapipe.autoflip; - -// Options for the VisualScorer module. -// Next tag: 6 -message VisualScorerOptions { - // Weights for the various cues. A larger weight means that the corresponding - // cue will be of higher importance when generating the combined score. - optional float area_weight = 1 [default = 1.0]; - // Sharpness is not yet implemented. - optional float sharpness_weight = 2 [default = 0.0]; - optional float colorfulness_weight = 3 [default = 0.0]; -} diff --git a/mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc b/mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc deleted file mode 100644 index f1a3bb5d1..000000000 --- a/mediapipe/examples/desktop/autoflip/quality/visual_scorer_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/examples/desktop/autoflip/quality/visual_scorer.h" - -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace autoflip { -namespace { - -TEST(VisualScorerTest, ScoresArea) { - cv::Mat image_mat(200, 200, CV_8UC3); - SalientRegion region = ParseTextProtoOrDie( - R"pb(location { x: 10 y: 10 width: 100 height: 100 })pb"); - - VisualScorerOptions options = ParseTextProtoOrDie( - R"pb(area_weight: 1.0 sharpness_weight: 0 colorfulness_weight: 0)pb"); - VisualScorer scorer(options); - float score = 0.0; - MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score)); - EXPECT_EQ(0.25, score); // (100 * 100) / (200 * 200). -} - -TEST(VisualScorerTest, ScoresSharpness) { - SalientRegion region = ParseTextProtoOrDie( - R"pb(location { x: 10 y: 10 width: 100 height: 100 })pb"); - - VisualScorerOptions options = ParseTextProtoOrDie( - R"pb(area_weight: 0 sharpness_weight: 1.0 colorfulness_weight: 0)pb"); - VisualScorer scorer(options); - - // Compute the score of an empty image and an image with a rectangle. - cv::Mat image_mat(200, 200, CV_8UC3); - image_mat.setTo(cv::Scalar(0, 0, 0)); - float score_rect = 0; - auto status = scorer.CalculateScore(image_mat, region, &score_rect); - EXPECT_EQ(status.code(), StatusCode::kInvalidArgument); -} - -TEST(VisualScorerTest, ScoresColorfulness) { - SalientRegion region = ParseTextProtoOrDie( - R"pb(location { x: 10 y: 10 width: 50 height: 150 })pb"); - - VisualScorerOptions options = ParseTextProtoOrDie( - R"pb(area_weight: 0 sharpness_weight: 0 colorfulness_weight: 1.0)pb"); - VisualScorer scorer(options); - - // Compute the scores of images with 1, 2 and 3 colors. - cv::Mat image_mat(200, 200, CV_8UC3); - image_mat.setTo(cv::Scalar(0, 0, 255)); - float score_1c = 0, score_2c = 0, score_3c = 0; - MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_1c)); - image_mat(cv::Rect(30, 30, 20, 20)).setTo(cv::Scalar(128, 0, 0)); - MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_2c)); - image_mat(cv::Rect(50, 50, 20, 20)).setTo(cv::Scalar(255, 128, 0)); - MP_EXPECT_OK(scorer.CalculateScore(image_mat, region, &score_3c)); - // Images with more colors should have a higher score. - EXPECT_LT(score_1c, score_2c); - EXPECT_LT(score_2c, score_3c); -} - -} // namespace -} // namespace autoflip -} // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/subgraph/BUILD b/mediapipe/examples/desktop/autoflip/subgraph/BUILD deleted file mode 100644 index 6c3e2616c..000000000 --- a/mediapipe/examples/desktop/autoflip/subgraph/BUILD +++ /dev/null @@ -1,56 +0,0 @@ -load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph") - -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -FACE_DETECTION_DEPS = [ - "//mediapipe/calculators/image:image_transformation_calculator", - "//mediapipe/calculators/tflite:ssd_anchors_calculator", - "//mediapipe/calculators/tflite:tflite_converter_calculator", - "//mediapipe/calculators/tflite:tflite_inference_calculator", - "//mediapipe/calculators/tflite:tflite_tensors_to_detections_calculator", - "//mediapipe/calculators/util:detection_label_id_to_text_calculator", - "//mediapipe/calculators/util:detection_letterbox_removal_calculator", - "//mediapipe/calculators/util:non_max_suppression_calculator", -] - -mediapipe_simple_subgraph( - name = "autoflip_face_detection_subgraph", - graph = "face_detection_subgraph.pbtxt", - register_as = "AutoFlipFaceDetectionSubgraph", - visibility = ["//visibility:public"], - deps = FACE_DETECTION_DEPS, -) - -mediapipe_simple_subgraph( - name = "autoflip_front_face_detection_subgraph", - graph = "front_face_detection_subgraph.pbtxt", - register_as = "AutoFlipFrontFaceDetectionSubgraph", - visibility = ["//visibility:public"], - deps = FACE_DETECTION_DEPS, -) - -mediapipe_simple_subgraph( - name = "autoflip_object_detection_subgraph", - graph = "autoflip_object_detection_subgraph.pbtxt", - register_as = "AutoFlipObjectDetectionSubgraph", - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/graphs/object_detection:desktop_tflite_calculators", - ], -) diff --git a/mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt b/mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt deleted file mode 100644 index bd2e7a7d4..000000000 --- a/mediapipe/examples/desktop/autoflip/subgraph/autoflip_object_detection_subgraph.pbtxt +++ /dev/null @@ -1,126 +0,0 @@ -# MediaPipe graph that performs object detection with TensorFlow Lite on CPU. - -input_stream: "VIDEO:input_video" -output_stream: "DETECTIONS:output_detections" - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video" - options: { - [mediapipe.ImageTransformationCalculatorOptions.ext] { - output_width: 320 - output_height: 320 - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:image_tensor" -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - options: { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/models/ssdlite_object_detection.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options: { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 6 - min_scale: 0.2 - max_scale: 0.95 - input_size_height: 320 - input_size_width: 320 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 16 - strides: 32 - strides: 64 - strides: 128 - strides: 256 - strides: 512 - aspect_ratios: 1.0 - aspect_ratios: 2.0 - aspect_ratios: 0.5 - aspect_ratios: 3.0 - aspect_ratios: 0.3333 - reduce_boxes_in_lowest_layer: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { - num_classes: 91 - num_boxes: 2034 - num_coords: 4 - ignore_classes: 0 - sigmoid_score: true - apply_exponential_on_box_size: true - x_scale: 10.0 - y_scale: 10.0 - h_scale: 5.0 - w_scale: 5.0 - min_score_thresh: 0.6 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - options: { - [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { - min_suppression_threshold: 0.4 - max_num_detections: 5 - overlap_type: INTERSECTION_OVER_UNION - return_empty_detections: true - } - } -} - -# Maps detection label IDs to the corresponding label text. The label map is -# provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "output_detections" - options: { - [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] { - label_map_path: "mediapipe/models/ssdlite_object_detection_labelmap.txt" - } - } -} diff --git a/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt b/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt deleted file mode 100644 index 4024f355a..000000000 --- a/mediapipe/examples/desktop/autoflip/subgraph/face_detection_subgraph.pbtxt +++ /dev/null @@ -1,121 +0,0 @@ -# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. - -input_stream: "VIDEO:input_video" -output_stream: "DETECTIONS:output_detections" - - -# Transforms the input image on CPU to a 128x128 image. To scale the input -# image, the scale_mode option is set to FIT to preserve the aspect ratio, -# resulting in potential letterboxing in the transformed image. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "LETTERBOX_PADDING:letterbox_padding" - options: { - [mediapipe.ImageTransformationCalculatorOptions.ext] { - output_width: 192 - output_height: 192 - scale_mode: FIT - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "TENSORS:image_tensor" -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - options: { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "mediapipe/modules/face_detection/face_detection_back.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options: { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 1 - min_scale: 0.1484375 - max_scale: 0.75 - input_size_height: 192 - input_size_width: 192 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 4 - aspect_ratios: 1.0 - fixed_anchor_size: true - interpolated_scale_aspect_ratio: 0.0 - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { - num_classes: 1 - num_boxes: 2304 - num_coords: 16 - box_coord_offset: 0 - keypoint_coord_offset: 4 - num_keypoints: 6 - num_values_per_keypoint: 2 - sigmoid_score: true - score_clipping_thresh: 100.0 - reverse_output_order: true - x_scale: 192.0 - y_scale: 192.0 - h_scale: 192.0 - w_scale: 192.0 - min_score_thresh: 0.6 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - options: { - [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { - min_suppression_threshold: 0.3 - overlap_type: INTERSECTION_OVER_UNION - algorithm: WEIGHTED - return_empty_detections: true - } - } -} - -# Adjusts detection locations (already normalized to [0.f, 1.f]) on the -# letterboxed image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (the -# input image to the graph before image transformation). -node { - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:filtered_detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:output_detections" -} diff --git a/mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt b/mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt deleted file mode 100644 index 3b2d410f5..000000000 --- a/mediapipe/examples/desktop/autoflip/subgraph/front_face_detection_subgraph.pbtxt +++ /dev/null @@ -1,135 +0,0 @@ -# MediaPipe graph that performs face detection with TensorFlow Lite on CPU. Model paths setup for web use. -# TODO: parameterize input paths to support desktop use, for web only. -input_stream: "VIDEO:input_video" -output_stream: "DETECTIONS:output_detections" - -# Transforms the input image on CPU to a 128x128 image. To scale the input -# image, the scale_mode option is set to FIT to preserve the aspect ratio, -# resulting in potential letterboxing in the transformed image. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:input_video" - output_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "LETTERBOX_PADDING:letterbox_padding" - options: { - [mediapipe.ImageTransformationCalculatorOptions.ext] { - output_width: 128 - output_height: 128 - scale_mode: FIT - } - } -} - -# Converts the transformed input image on CPU into an image tensor stored as a -# TfLiteTensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video_cpu" - output_stream: "TENSORS:image_tensor" -} - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:image_tensor" - output_stream: "TENSORS:detection_tensors" - options: { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - model_path: "face_detection_front.tflite" - } - } -} - -# Generates a single side packet containing a vector of SSD anchors based on -# the specification in the options. -node { - calculator: "SsdAnchorsCalculator" - output_side_packet: "anchors" - options: { - [mediapipe.SsdAnchorsCalculatorOptions.ext] { - num_layers: 4 - min_scale: 0.1484375 - max_scale: 0.75 - input_size_height: 128 - input_size_width: 128 - anchor_offset_x: 0.5 - anchor_offset_y: 0.5 - strides: 8 - strides: 16 - strides: 16 - strides: 16 - aspect_ratios: 1.0 - fixed_anchor_size: true - } - } -} - -# Decodes the detection tensors generated by the TensorFlow Lite model, based on -# the SSD anchors and the specification in the options, into a vector of -# detections. Each detection describes a detected object. -node { - calculator: "TfLiteTensorsToDetectionsCalculator" - input_stream: "TENSORS:detection_tensors" - input_side_packet: "ANCHORS:anchors" - output_stream: "DETECTIONS:detections" - options: { - [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] { - num_classes: 1 - num_boxes: 896 - num_coords: 16 - box_coord_offset: 0 - keypoint_coord_offset: 4 - num_keypoints: 6 - num_values_per_keypoint: 2 - sigmoid_score: true - score_clipping_thresh: 100.0 - reverse_output_order: true - x_scale: 128.0 - y_scale: 128.0 - h_scale: 128.0 - w_scale: 128.0 - min_score_thresh: 0.75 - } - } -} - -# Performs non-max suppression to remove excessive detections. -node { - calculator: "NonMaxSuppressionCalculator" - input_stream: "detections" - output_stream: "filtered_detections" - options: { - [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { - min_suppression_threshold: 0.3 - overlap_type: INTERSECTION_OVER_UNION - algorithm: WEIGHTED - return_empty_detections: true - } - } -} - -# Maps detection label IDs to the corresponding label text ("Face"). The label -# map is provided in the label_map_path option. -node { - calculator: "DetectionLabelIdToTextCalculator" - input_stream: "filtered_detections" - output_stream: "labeled_detections" - options: { - [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] { - label_map_path: "face_detection_front_labelmap.txt" - } - } -} - -# Adjusts detection locations (already normalized to [0.f, 1.f]) on the -# letterboxed image (after image transformation with the FIT scale mode) to the -# corresponding locations on the same image with the letterbox removed (the -# input image to the graph before image transformation). -node { - calculator: "DetectionLetterboxRemovalCalculator" - input_stream: "DETECTIONS:labeled_detections" - input_stream: "LETTERBOX_PADDING:letterbox_padding" - output_stream: "DETECTIONS:output_detections" -} diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc deleted file mode 100644 index 0d26aa0d3..000000000 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// An example of sending OpenCV webcam frames into a MediaPipe graph. -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -constexpr char kInputStream[] = "input_video"; -constexpr char kOutputStream[] = "output_video"; -constexpr char kWindowName[] = "MediaPipe"; - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); -ABSL_FLAG(std::string, input_video_path, "", - "Full path of video to load. " - "If not provided, attempt to use a webcam."); -ABSL_FLAG(std::string, output_video_path, "", - "Full path of where to save result (.mp4 only). " - "If not provided, show result in a window."); - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config)); - - LOG(INFO) << "Initialize the camera or load the video."; - cv::VideoCapture capture; - const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); - if (load_video) { - capture.open(absl::GetFlag(FLAGS_input_video_path)); - } else { - capture.open(0); - } - RET_CHECK(capture.isOpened()); - - cv::VideoWriter writer; - const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty(); - if (!save_video) { - cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); -#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) - capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); - capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); - capture.set(cv::CAP_PROP_FPS, 30); -#endif - } - - LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, - graph.AddOutputStreamPoller(kOutputStream)); - MP_RETURN_IF_ERROR(graph.StartRun({})); - - LOG(INFO) << "Start grabbing and processing frames."; - bool grab_frames = true; - while (grab_frames) { - // Capture opencv camera or video frame. - cv::Mat camera_frame_raw; - capture >> camera_frame_raw; - if (camera_frame_raw.empty()) { - if (!load_video) { - LOG(INFO) << "Ignore empty frames from camera."; - continue; - } - LOG(INFO) << "Empty frame, end of video reached."; - break; - } - cv::Mat camera_frame; - cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); - if (!load_video) { - cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - } - - // Wrap Mat into an ImageFrame. - auto input_frame = absl::make_unique( - mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, - mediapipe::ImageFrame::kDefaultAlignmentBoundary); - cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); - camera_frame.copyTo(input_frame_mat); - - // Send image packet into the graph. - size_t frame_timestamp_us = - (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; - MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( - kInputStream, mediapipe::Adopt(input_frame.release()) - .At(mediapipe::Timestamp(frame_timestamp_us)))); - - // Get the graph result packet, or stop if that fails. - mediapipe::Packet packet; - if (!poller.Next(&packet)) break; - auto& output_frame = packet.Get(); - - // Convert back to opencv for display or saving. - cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); - cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); - if (save_video) { - if (!writer.isOpened()) { - LOG(INFO) << "Prepare video writer."; - writer.open(absl::GetFlag(FLAGS_output_video_path), - mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 - capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); - RET_CHECK(writer.isOpened()); - } - writer.write(output_frame_mat); - } else { - cv::imshow(kWindowName, output_frame_mat); - // Press any key to exit. - const int pressed_key = cv::waitKey(5); - if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; - } - } - - LOG(INFO) << "Shutting down."; - if (writer.isOpened()) writer.release(); - MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); - return graph.WaitUntilDone(); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc b/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc deleted file mode 100644 index 586565db4..000000000 --- a/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// An example of sending OpenCV webcam frames into a MediaPipe graph. -// This example requires a linux computer and a GPU with EGL support drivers. -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/gpu/gpu_shared_data_internal.h" - -constexpr char kInputStream[] = "input_video"; -constexpr char kOutputStream[] = "output_video"; -constexpr char kWindowName[] = "MediaPipe"; - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); -ABSL_FLAG(std::string, input_video_path, "", - "Full path of video to load. " - "If not provided, attempt to use a webcam."); -ABSL_FLAG(std::string, output_video_path, "", - "Full path of where to save result (.mp4 only). " - "If not provided, show result in a window."); - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config)); - - LOG(INFO) << "Initialize the GPU."; - ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create()); - MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources))); - mediapipe::GlCalculatorHelper gpu_helper; - gpu_helper.InitializeForTest(graph.GetGpuResources().get()); - - LOG(INFO) << "Initialize the camera or load the video."; - cv::VideoCapture capture; - const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); - if (load_video) { - capture.open(absl::GetFlag(FLAGS_input_video_path)); - } else { - capture.open(0); - } - RET_CHECK(capture.isOpened()); - - cv::VideoWriter writer; - const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty(); - if (!save_video) { - cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); -#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) - capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); - capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); - capture.set(cv::CAP_PROP_FPS, 30); -#endif - } - - LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, - graph.AddOutputStreamPoller(kOutputStream)); - MP_RETURN_IF_ERROR(graph.StartRun({})); - - LOG(INFO) << "Start grabbing and processing frames."; - bool grab_frames = true; - while (grab_frames) { - // Capture opencv camera or video frame. - cv::Mat camera_frame_raw; - capture >> camera_frame_raw; - if (camera_frame_raw.empty()) { - if (!load_video) { - LOG(INFO) << "Ignore empty frames from camera."; - continue; - } - LOG(INFO) << "Empty frame, end of video reached."; - break; - } - cv::Mat camera_frame; - cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGBA); - if (!load_video) { - cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - } - - // Wrap Mat into an ImageFrame. - auto input_frame = absl::make_unique( - mediapipe::ImageFormat::SRGBA, camera_frame.cols, camera_frame.rows, - mediapipe::ImageFrame::kGlDefaultAlignmentBoundary); - cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); - camera_frame.copyTo(input_frame_mat); - - // Prepare and add graph input packet. - size_t frame_timestamp_us = - (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; - MP_RETURN_IF_ERROR( - gpu_helper.RunInGlContext([&input_frame, &frame_timestamp_us, &graph, - &gpu_helper]() -> absl::Status { - // Convert ImageFrame to GpuBuffer. - auto texture = gpu_helper.CreateSourceTexture(*input_frame.get()); - auto gpu_frame = texture.GetFrame(); - glFlush(); - texture.Release(); - // Send GPU image packet into the graph. - MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( - kInputStream, mediapipe::Adopt(gpu_frame.release()) - .At(mediapipe::Timestamp(frame_timestamp_us)))); - return absl::OkStatus(); - })); - - // Get the graph result packet, or stop if that fails. - mediapipe::Packet packet; - if (!poller.Next(&packet)) break; - std::unique_ptr output_frame; - - // Convert GpuBuffer to ImageFrame. - MP_RETURN_IF_ERROR(gpu_helper.RunInGlContext( - [&packet, &output_frame, &gpu_helper]() -> absl::Status { - auto& gpu_frame = packet.Get(); - auto texture = gpu_helper.CreateSourceTexture(gpu_frame); - output_frame = absl::make_unique( - mediapipe::ImageFormatForGpuBufferFormat(gpu_frame.format()), - gpu_frame.width(), gpu_frame.height(), - mediapipe::ImageFrame::kGlDefaultAlignmentBoundary); - gpu_helper.BindFramebuffer(texture); - const auto info = mediapipe::GlTextureInfoForGpuBufferFormat( - gpu_frame.format(), 0, gpu_helper.GetGlVersion()); - glReadPixels(0, 0, texture.width(), texture.height(), info.gl_format, - info.gl_type, output_frame->MutablePixelData()); - glFlush(); - texture.Release(); - return absl::OkStatus(); - })); - - // Convert back to opencv for display or saving. - cv::Mat output_frame_mat = mediapipe::formats::MatView(output_frame.get()); - if (output_frame_mat.channels() == 4) - cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGBA2BGR); - else - cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); - if (save_video) { - if (!writer.isOpened()) { - LOG(INFO) << "Prepare video writer."; - writer.open(absl::GetFlag(FLAGS_output_video_path), - mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 - capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); - RET_CHECK(writer.isOpened()); - } - writer.write(output_frame_mat); - } else { - cv::imshow(kWindowName, output_frame_mat); - // Press any key to exit. - const int pressed_key = cv::waitKey(5); - if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; - } - } - - LOG(INFO) << "Shutting down."; - if (writer.isOpened()) writer.release(); - MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); - return graph.WaitUntilDone(); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/face_detection/BUILD b/mediapipe/examples/desktop/face_detection/BUILD deleted file mode 100644 index 5743ae788..000000000 --- a/mediapipe/examples/desktop/face_detection/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "face_detection_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/face_detection:desktop_live_calculators", - ], -) - -# Linux only -cc_binary( - name = "face_detection_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/face_detection:desktop_live_gpu_calculators", - ], -) diff --git a/mediapipe/examples/desktop/face_mesh/BUILD b/mediapipe/examples/desktop/face_mesh/BUILD deleted file mode 100644 index c63814804..000000000 --- a/mediapipe/examples/desktop/face_mesh/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "face_mesh_tflite", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/face_mesh:desktop_calculators", - ], -) - -cc_binary( - name = "face_mesh_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/face_mesh:desktop_live_calculators", - ], -) - -# Linux only -cc_binary( - name = "face_mesh_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/face_mesh:desktop_live_gpu_calculators", - ], -) diff --git a/mediapipe/examples/desktop/hair_segmentation/BUILD b/mediapipe/examples/desktop/hair_segmentation/BUILD deleted file mode 100644 index 9b799f347..000000000 --- a/mediapipe/examples/desktop/hair_segmentation/BUILD +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -# Linux only -cc_binary( - name = "hair_segmentation_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/hair_segmentation:mobile_calculators", - ], -) - -cc_binary( - name = "hair_segmentation_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - ] + select({ - "//mediapipe/gpu:disable_gpu": [ - "//mediapipe/graphs/hair_segmentation:desktop_calculators", - ], - "//conditions:default": [ - "//mediapipe/graphs/hair_segmentation:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/desktop/hand_tracking/BUILD b/mediapipe/examples/desktop/hand_tracking/BUILD deleted file mode 100644 index da6eef456..000000000 --- a/mediapipe/examples/desktop/hand_tracking/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "hand_tracking_tflite", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/hand_tracking:desktop_tflite_calculators", - ], -) - -cc_binary( - name = "hand_tracking_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/hand_tracking:desktop_tflite_calculators", - ], -) - -# Linux only -cc_binary( - name = "hand_tracking_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/hand_tracking:mobile_calculators", - ], -) diff --git a/mediapipe/examples/desktop/hello_world/BUILD b/mediapipe/examples/desktop/hello_world/BUILD deleted file mode 100644 index edf98bf13..000000000 --- a/mediapipe/examples/desktop/hello_world/BUILD +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "hello_world", - srcs = ["hello_world.cc"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework:calculator_graph", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) diff --git a/mediapipe/examples/desktop/hello_world/hello_world.cc b/mediapipe/examples/desktop/hello_world/hello_world.cc deleted file mode 100644 index d7416e12a..000000000 --- a/mediapipe/examples/desktop/hello_world/hello_world.cc +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// A simple example to print out "Hello World!" from a MediaPipe graph. - -#include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -namespace mediapipe { - -absl::Status PrintHelloWorld() { - // Configures a simple graph, which concatenates 2 PassThroughCalculators. - CalculatorGraphConfig config = - ParseTextProtoOrDie(R"pb( - input_stream: "in" - output_stream: "out" - node { - calculator: "PassThroughCalculator" - input_stream: "in" - output_stream: "out1" - } - node { - calculator: "PassThroughCalculator" - input_stream: "out1" - output_stream: "out" - } - )pb"); - - CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config)); - ASSIGN_OR_RETURN(OutputStreamPoller poller, - graph.AddOutputStreamPoller("out")); - MP_RETURN_IF_ERROR(graph.StartRun({})); - // Give 10 input packets that contains the same std::string "Hello World!". - for (int i = 0; i < 10; ++i) { - MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( - "in", MakePacket("Hello World!").At(Timestamp(i)))); - } - // Close the input stream "in". - MP_RETURN_IF_ERROR(graph.CloseInputStream("in")); - mediapipe::Packet packet; - // Get the output packets std::string. - while (poller.Next(&packet)) { - LOG(INFO) << packet.Get(); - } - return graph.WaitUntilDone(); -} -} // namespace mediapipe - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - CHECK(mediapipe::PrintHelloWorld().ok()); - return 0; -} diff --git a/mediapipe/examples/desktop/holistic_tracking/BUILD b/mediapipe/examples/desktop/holistic_tracking/BUILD deleted file mode 100644 index 0f69c1e4f..000000000 --- a/mediapipe/examples/desktop/holistic_tracking/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "holistic_tracking_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/holistic_tracking:holistic_tracking_cpu_graph_deps", - ], -) - -# Linux only -cc_binary( - name = "holistic_tracking_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu_deps", - ], -) diff --git a/mediapipe/examples/desktop/iris_tracking/BUILD b/mediapipe/examples/desktop/iris_tracking/BUILD deleted file mode 100644 index c6596de0b..000000000 --- a/mediapipe/examples/desktop/iris_tracking/BUILD +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "iris_depth_from_image_desktop", - srcs = ["iris_depth_from_image_desktop.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:opencv_highgui", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/graphs/iris_tracking:iris_depth_cpu_deps", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - ], -) - -cc_binary( - name = "iris_tracking_cpu_video_input", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_video_input_deps", - ], -) - -cc_binary( - name = "iris_tracking_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps", - ], -) - -# Linux only -cc_binary( - name = "iris_tracking_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps", - ], -) diff --git a/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc b/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc deleted file mode 100644 index 928ebb207..000000000 --- a/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// A utility to extract iris depth from a single image of face using the graph -// mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt. -#include -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_video_inc.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -constexpr char kInputStream[] = "input_image_bytes"; -constexpr char kOutputImageStream[] = "output_image"; -constexpr char kLeftIrisDepthMmStream[] = "left_iris_depth_mm"; -constexpr char kRightIrisDepthMmStream[] = "right_iris_depth_mm"; -constexpr char kWindowName[] = "MediaPipe"; -constexpr char kCalculatorGraphConfigFile[] = - "mediapipe/graphs/iris_tracking/iris_depth_cpu.pbtxt"; -constexpr float kMicrosPerSecond = 1e6; - -ABSL_FLAG(std::string, input_image_path, "", - "Full path of image to load. " - "If not provided, nothing will run."); -ABSL_FLAG(std::string, output_image_path, "", - "Full path of where to save image result (.jpg only). " - "If not provided, show result in a window."); - -namespace { - -absl::StatusOr ReadFileToString(const std::string& file_path) { - std::string contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents(file_path, &contents)); - return contents; -} - -absl::Status ProcessImage(std::unique_ptr graph) { - LOG(INFO) << "Load the image."; - ASSIGN_OR_RETURN(const std::string raw_image, - ReadFileToString(absl::GetFlag(FLAGS_input_image_path))); - - LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller output_image_poller, - graph->AddOutputStreamPoller(kOutputImageStream)); - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller left_iris_depth_poller, - graph->AddOutputStreamPoller(kLeftIrisDepthMmStream)); - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller right_iris_depth_poller, - graph->AddOutputStreamPoller(kRightIrisDepthMmStream)); - MP_RETURN_IF_ERROR(graph->StartRun({})); - - // Send image packet into the graph. - const size_t fake_timestamp_us = (double)cv::getTickCount() / - (double)cv::getTickFrequency() * - kMicrosPerSecond; - MP_RETURN_IF_ERROR(graph->AddPacketToInputStream( - kInputStream, mediapipe::MakePacket(raw_image).At( - mediapipe::Timestamp(fake_timestamp_us)))); - - // Get the graph result packets, or stop if that fails. - mediapipe::Packet left_iris_depth_packet; - if (!left_iris_depth_poller.Next(&left_iris_depth_packet)) { - return absl::UnknownError( - "Failed to get packet from output stream 'left_iris_depth_mm'."); - } - const auto& left_iris_depth_mm = left_iris_depth_packet.Get(); - const int left_iris_depth_cm = std::round(left_iris_depth_mm / 10); - std::cout << "Left Iris Depth: " << left_iris_depth_cm << " cm." << std::endl; - - mediapipe::Packet right_iris_depth_packet; - if (!right_iris_depth_poller.Next(&right_iris_depth_packet)) { - return absl::UnknownError( - "Failed to get packet from output stream 'right_iris_depth_mm'."); - } - const auto& right_iris_depth_mm = right_iris_depth_packet.Get(); - const int right_iris_depth_cm = std::round(right_iris_depth_mm / 10); - std::cout << "Right Iris Depth: " << right_iris_depth_cm << " cm." - << std::endl; - - mediapipe::Packet output_image_packet; - if (!output_image_poller.Next(&output_image_packet)) { - return absl::UnknownError( - "Failed to get packet from output stream 'output_image'."); - } - auto& output_frame = output_image_packet.Get(); - - // Convert back to opencv for display or saving. - cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); - cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); - const bool save_image = !absl::GetFlag(FLAGS_output_image_path).empty(); - if (save_image) { - LOG(INFO) << "Saving image to file..."; - cv::imwrite(absl::GetFlag(FLAGS_output_image_path), output_frame_mat); - } else { - cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); - cv::imshow(kWindowName, output_frame_mat); - // Press any key to exit. - cv::waitKey(0); - } - - LOG(INFO) << "Shutting down."; - MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream)); - return graph->WaitUntilDone(); -} - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - kCalculatorGraphConfigFile, &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - - LOG(INFO) << "Initialize the calculator graph."; - std::unique_ptr graph = - absl::make_unique(); - MP_RETURN_IF_ERROR(graph->Initialize(config)); - - const bool load_image = !absl::GetFlag(FLAGS_input_image_path).empty(); - if (load_image) { - return ProcessImage(std::move(graph)); - } else { - return absl::InvalidArgumentError("Missing image file."); - } -} - -} // namespace - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/media_sequence/BUILD b/mediapipe/examples/desktop/media_sequence/BUILD deleted file mode 100644 index 1a88aa109..000000000 --- a/mediapipe/examples/desktop/media_sequence/BUILD +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_library( - name = "run_graph_file_io_main", - srcs = ["run_graph_file_io_main.cc"], - deps = [ - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - "@com_google_absl//absl/strings", - ], -) - -cc_binary( - name = "media_sequence_demo", - deps = [ - ":run_graph_file_io_main", - "//mediapipe/graphs/media_sequence:clipped_images_from_file_at_24fps_calculators", - "//mediapipe/graphs/media_sequence:tvl1_flow_and_rgb_from_file_calculators", - ], -) diff --git a/mediapipe/examples/desktop/media_sequence/README.md b/mediapipe/examples/desktop/media_sequence/README.md deleted file mode 100644 index 828d152e4..000000000 --- a/mediapipe/examples/desktop/media_sequence/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# Preparing data sets for machine learning with MediaPipe -We include two pipelines to prepare data sets for training TensorFlow models. - -Using these data sets is split into two parts. First, the data set is -constructed in with a Python script and MediaPipe C++ binary. The C++ binary -should be compiled by the end user because the preparation for different data -sets requires different MediaPipe calculator dependencies. The result of running -the script is a data set of TFRecord files on disk. The second stage is reading -the data from TensorFlow into a tf.data.Dataset. Both pipelines can be imported -and support a simple call to as_dataset() to make the data available. - -### Demo data set -To generate the demo dataset you must have Tensorflow installed. Then the -media_sequence_demo binary must be built from the top directory in the mediapipe -repo and the command to build the data set must be run from the same directory. -``` -bazel build -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo \ - --define MEDIAPIPE_DISABLE_GPU=1 - -python -m mediapipe.examples.desktop.media_sequence.demo_dataset \ - --alsologtostderr \ - --path_to_demo_data=/tmp/demo_data/ \ - --path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/\ -media_sequence/media_sequence_demo \ - --path_to_graph_directory=mediapipe/graphs/media_sequence/ -``` - -### Charades data set - -The Charades data set is ready for training and/or evaluating action recognition -models in TensorFlow. You may only use this script in ways that comply with the -Allen Institute for Artificial Intelligence's [license for the Charades data -set.](https://allenai.org/plato/charades/license.txt) - -To generate the Charades dataset you must have Tensorflow installed. Then the -media_sequence_demo binary must be built from the top directory in the mediapipe -repo and the command to build the data set must be run from the same directory. - -``` -bazel build -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo \ - --define MEDIAPIPE_DISABLE_GPU=1 - -python -m mediapipe.examples.desktop.media_sequence.charades_dataset \ - --alsologtostderr \ - --path_to_charades_data=/tmp/charades_data/ \ - --path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/\ -media_sequence/media_sequence_demo \ - --path_to_graph_directory=mediapipe/graphs/media_sequence/ -``` - -### Custom videos in the Kinetics format - -To produce data in the same format at the Kinetics data, use the kinetics.py -script. - -To generate the dataset you must have Tensorflow installed. Then the -media_sequence_demo binary must be built from the top directory in the mediapipe -repo and the command to build the data set must be run from the same directory. - -``` -echo "Credit for this video belongs to: ESA/Hubble; Music: Johan B. Monell" -wget https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1608c.mp4 -O /tmp/heic1608c.mp4 -CUSTOM_CSV=/tmp/custom_kinetics.csv -VIDEO_PATH=/tmp/heic1608c.mp4 -echo -e "video,time_start,time_end,split\n${VIDEO_PATH},0,10,custom" > ${CUSTOM_CSV} - -bazel build -c opt mediapipe/examples/desktop/media_sequence:media_sequence_demo \ - --define MEDIAPIPE_DISABLE_GPU=1 - -python -m mediapipe.examples.desktop.media_sequence.kinetics_dataset \ - --alsologtostderr \ - --splits_to_process=custom \ - --path_to_custom_csv=${CUSTOM_CSV} \ - --video_path_format_string={video} \ - --path_to_kinetics_data=/tmp/ms/kinetics/ \ - --path_to_mediapipe_binary=bazel-bin/mediapipe/examples/desktop/\ -media_sequence/media_sequence_demo \ - --path_to_graph_directory=mediapipe/graphs/media_sequence/ -``` diff --git a/mediapipe/examples/desktop/media_sequence/__init__.py b/mediapipe/examples/desktop/media_sequence/__init__.py deleted file mode 100644 index 6db73bc52..000000000 --- a/mediapipe/examples/desktop/media_sequence/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Copyright 2019 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/mediapipe/examples/desktop/media_sequence/charades_dataset.py b/mediapipe/examples/desktop/media_sequence/charades_dataset.py deleted file mode 100644 index 9c8540575..000000000 --- a/mediapipe/examples/desktop/media_sequence/charades_dataset.py +++ /dev/null @@ -1,519 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""Code to download and parse the Charades dataset for TensorFlow models. - -The [Charades data set](https://allenai.org/plato/charades/) is a data set of -human action recognition collected with and maintained by the Allen Institute -for Artificial Intelligence. This script downloads and prepares the data set for -training a TensorFlow model. To use this script, you must abide by the -[lincense](https://allenai.org/plato/charades/license.txt) for the Charades data -set provided by the Allen Institute. The license for this script only covers -this code and not the data set. - -Running this code as a module generates the data set on disk. First, the -required files are downloaded (_download_data). Then, for each split in the -data set (generate_examples), the metadata is generated from the annotations for -each example (_generate_metadata), and MediaPipe is used to fill in the video -frames (_run_mediapipe). The data set is written to disk as a set of numbered -TFRecord files. If the download is disrupted, the incomplete files will need to -be removed before running the script again. This pattern can be reproduced and -modified to generate most video data sets. - -Generating the data on disk will probably take 4-8 hours and requires 150 GB of -disk space. (Image compression quality is the primary determiner of disk usage.) -After generating the data, the 30 GB of compressed video data can be deleted. - -Once the data is on disk, reading the data as a tf.data.Dataset is accomplished -with the following lines: - - charades = CharadesDataset("charades_data_path") - dataset = charades.as_dataset("test") - # implement additional processing and batching here - images_and_labels = dataset.make_one_shot_iterator().get_next() - images = images_and_labels["images"] - labels = image_and_labels["classification_target"] - label_weights = image_and_labels["indicator_matrix"] - -This data is structured for per-frame action classification where images is -the sequence of images, labels are the sequence of classification targets and, -label_weights is 1 for valid frames and 0 for padded frames (if any). See -as_dataset() for more details. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import csv -import os -import random -import subprocess -import sys -import tempfile -import zipfile - -from absl import app -from absl import flags -from absl import logging -from six.moves import range -from six.moves import urllib -import tensorflow.compat.v1 as tf - -from mediapipe.util.sequence import media_sequence as ms - - -DATA_URL_ANNOTATIONS = "http://ai2-website.s3.amazonaws.com/data/Charades.zip" -DATA_URL_VIDEOS = "http://ai2-website.s3.amazonaws.com/data/Charades_v1_480.zip" -DATA_URL_LICENSE = "https://allenai.org/plato/charades/license.txt" -CITATION = r"""@article{sigurdsson2016hollywood, -author = {Gunnar A. Sigurdsson and G{\"u}l Varol and Xiaolong Wang and Ivan Laptev and Ali Farhadi and Abhinav Gupta}, -title = {Hollywood in Homes: Crowdsourcing Data Collection for Activity Understanding}, -journal = {ArXiv e-prints}, -eprint = {1604.01753}, -year = {2016}, -url = {http://arxiv.org/abs/1604.01753}, -}""" -SECONDS_TO_MICROSECONDS = 1000000 -GRAPHS = ["clipped_images_from_file_at_24fps.pbtxt"] -SPLITS = { - "train": ("charades_v1_train_records", # base name for sharded files - "Charades_v1_train.csv", # path to csv of annotations - 1000, # number of shards - 7986), # number of examples - "test": ("charades_v1_test_records", - "Charades_v1_test.csv", - 100, - 1864), -} -NUM_CLASSES = 157 -CLASS_LABEL_OFFSET = 1 - - -class Charades(object): - """Generates and loads the Charades data set.""" - - def __init__(self, path_to_data): - if not path_to_data: - raise ValueError("You must supply the path to the data directory.") - self.path_to_data = path_to_data - - def as_dataset(self, split, shuffle=False, repeat=False, - serialized_prefetch_size=32, decoded_prefetch_size=32): - """Returns Charades as a tf.data.Dataset. - - After running this function, calling padded_batch() on the Dataset object - will produce batches of data, but additional preprocessing may be desired. - If using padded_batch, the indicator_matrix output distinguishes valid - from padded frames. - - Args: - split: either "train" or "test" - shuffle: if true, shuffles both files and examples. - repeat: if true, repeats the data set forever. - serialized_prefetch_size: the buffer size for reading from disk. - decoded_prefetch_size: the buffer size after decoding. - Returns: - A tf.data.Dataset object with the following structure: { - "images": uint8 tensor, shape [time, height, width, channels] - "segment_matrix": binary tensor of segments, shape [time, num_segments]. - See one_hot_segments() for details. - "indicator_matrix": binary tensor indicating valid frames, - shape [time, 1]. If padded with zeros to align sizes, the indicator - marks where segments is valid. - "classification_target": binary tensor of classification targets, - shape [time, 158 classes]. More than one value in a row can be 1.0 if - segments overlap. - "example_id": a unique string id for each example, shape []. - "sampling_rate": the frame rate for each sequence, shape []. - "gt_segment_seconds": the start and end time of each segment, - shape [num_segments, 2]. - "gt_segment_classes": the class labels for each segment, - shape [num_segments]. - "num_segments": the number of segments in the example, shape []. - "num_timesteps": the number of timesteps in the example, shape []. - """ - def parse_fn(sequence_example): - """Parses a Charades example.""" - context_features = { - ms.get_example_id_key(): ms.get_example_id_default_parser(), - ms.get_segment_start_index_key(): ( - ms.get_segment_start_index_default_parser()), - ms.get_segment_end_index_key(): ( - ms.get_segment_end_index_default_parser()), - ms.get_segment_label_index_key(): ( - ms.get_segment_label_index_default_parser()), - ms.get_segment_label_string_key(): ( - ms.get_segment_label_string_default_parser()), - ms.get_segment_start_timestamp_key(): ( - ms.get_segment_start_timestamp_default_parser()), - ms.get_segment_end_timestamp_key(): ( - ms.get_segment_end_timestamp_default_parser()), - ms.get_image_frame_rate_key(): ( - ms.get_image_frame_rate_default_parser()), - } - - sequence_features = { - ms.get_image_encoded_key(): ms.get_image_encoded_default_parser() - } - parsed_context, parsed_sequence = tf.io.parse_single_sequence_example( - sequence_example, context_features, sequence_features) - - sequence_length = tf.shape(parsed_sequence[ms.get_image_encoded_key()])[0] - num_segments = tf.shape( - parsed_context[ms.get_segment_label_index_key()])[0] - # segments matrix and targets for training. - segments_matrix, indicator = one_hot_segments( - tf.sparse_tensor_to_dense( - parsed_context[ms.get_segment_start_index_key()]), - tf.sparse_tensor_to_dense( - parsed_context[ms.get_segment_end_index_key()]), - sequence_length) - - classification_target = timepoint_classification_target( - segments_matrix, - tf.sparse_tensor_to_dense( - parsed_context[ms.get_segment_label_index_key()] - ) + CLASS_LABEL_OFFSET, - NUM_CLASSES + CLASS_LABEL_OFFSET) - - # [segments, 2] start and end time in seconds. - gt_segment_seconds = tf.to_float(tf.concat( - [tf.expand_dims(tf.sparse_tensor_to_dense(parsed_context[ - ms.get_segment_start_timestamp_key()]), 1), - tf.expand_dims(tf.sparse_tensor_to_dense(parsed_context[ - ms.get_segment_end_timestamp_key()]), 1)], - 1)) / float(SECONDS_TO_MICROSECONDS) - gt_segment_classes = tf.sparse_tensor_to_dense(parsed_context[ - ms.get_segment_label_index_key()]) + CLASS_LABEL_OFFSET - example_id = parsed_context[ms.get_example_id_key()] - sampling_rate = parsed_context[ms.get_image_frame_rate_key()] - - images = tf.map_fn(tf.image.decode_jpeg, - parsed_sequence[ms.get_image_encoded_key()], - back_prop=False, - dtype=tf.uint8) - - output_dict = { - "segment_matrix": segments_matrix, - "indicator_matrix": indicator, - "classification_target": classification_target, - "example_id": example_id, - "sampling_rate": sampling_rate, - "gt_segment_seconds": gt_segment_seconds, - "gt_segment_classes": gt_segment_classes, - "num_segments": num_segments, - "num_timesteps": sequence_length, - "images": images, - } - return output_dict - - if split not in SPLITS: - raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys()))) - all_shards = tf.io.gfile.glob( - os.path.join(self.path_to_data, SPLITS[split][0] + "-*-of-*")) - random.shuffle(all_shards) - all_shards_dataset = tf.data.Dataset.from_tensor_slices(all_shards) - cycle_length = min(16, len(all_shards)) - dataset = all_shards_dataset.apply( - tf.contrib.data.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=cycle_length, - block_length=1, sloppy=True, - buffer_output_elements=serialized_prefetch_size)) - dataset = dataset.prefetch(serialized_prefetch_size) - if shuffle: - dataset = dataset.shuffle(serialized_prefetch_size) - if repeat: - dataset = dataset.repeat() - dataset = dataset.map(parse_fn) - dataset = dataset.prefetch(decoded_prefetch_size) - return dataset - - def generate_examples(self, - path_to_mediapipe_binary, path_to_graph_directory): - """Downloads data and generates sharded TFRecords. - - Downloads the data files, generates metadata, and processes the metadata - with MediaPipe to produce tf.SequenceExamples for training. The resulting - files can be read with as_dataset(). After running this function the - original data files can be deleted. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - path_to_graph_directory: Path to the directory with MediaPipe graphs in - mediapipe/graphs/media_sequence/. - """ - if not path_to_mediapipe_binary: - raise ValueError( - "You must supply the path to the MediaPipe binary for " - "mediapipe/examples/desktop/demo:media_sequence_demo.") - if not path_to_graph_directory: - raise ValueError( - "You must supply the path to the directory with MediaPipe graphs in " - "mediapipe/graphs/media_sequence/.") - logging.info("Downloading data.") - annotation_dir, video_dir = self._download_data() - for name, annotations, shards, _ in SPLITS.values(): - annotation_file = os.path.join( - annotation_dir, annotations) - logging.info("Generating metadata for split: %s", name) - all_metadata = list(self._generate_metadata(annotation_file, video_dir)) - random.seed(47) - random.shuffle(all_metadata) - shard_names = [os.path.join(self.path_to_data, name + "-%05d-of-%05d" % ( - i, shards)) for i in range(shards)] - writers = [tf.io.TFRecordWriter(shard_name) for shard_name in shard_names] - with _close_on_exit(writers) as writers: - for i, seq_ex in enumerate(all_metadata): - print("Processing example %d of %d (%d%%) \r" % ( - i, len(all_metadata), i * 100 / len(all_metadata)), end="") - for graph in GRAPHS: - graph_path = os.path.join(path_to_graph_directory, graph) - seq_ex = self._run_mediapipe( - path_to_mediapipe_binary, seq_ex, graph_path) - writers[i % len(writers)].write(seq_ex.SerializeToString()) - logging.info("Data extraction complete.") - - def _generate_metadata(self, annotations_file, video_dir): - """For each row in the annotation CSV, generates the corresponding metadata. - - Args: - annotations_file: path to the file of Charades CSV annotations. - video_dir: path to the directory of video files referenced by the - annotations. - Yields: - Each tf.SequenceExample of metadata, ready to pass to MediaPipe. - """ - with open(annotations_file, "r") as annotations: - reader = csv.DictReader(annotations) - for row in reader: - metadata = tf.train.SequenceExample() - filepath = os.path.join(video_dir, "%s.mp4" % row["id"]) - actions = row["actions"].split(";") - action_indices = [] - action_strings = [] - action_start_times = [] - action_end_times = [] - for action in actions: - if not action: - continue - string, start, end = action.split(" ") - action_indices.append(int(string[1:])) - action_strings.append(bytes23(string)) - action_start_times.append(int(float(start) * SECONDS_TO_MICROSECONDS)) - action_end_times.append(int(float(end) * SECONDS_TO_MICROSECONDS)) - ms.set_example_id(bytes23(row["id"]), metadata) - ms.set_clip_data_path(bytes23(filepath), metadata) - ms.set_clip_start_timestamp(0, metadata) - ms.set_clip_end_timestamp( - int(float(row["length"]) * SECONDS_TO_MICROSECONDS), metadata) - ms.set_segment_start_timestamp(action_start_times, metadata) - ms.set_segment_end_timestamp(action_end_times, metadata) - ms.set_segment_label_string(action_strings, metadata) - ms.set_segment_label_index(action_indices, metadata) - yield metadata - - def _download_data(self): - """Downloads and extracts data if not already available.""" - if sys.version_info >= (3, 0): - urlretrieve = urllib.request.urlretrieve - else: - urlretrieve = urllib.request.urlretrieve - logging.info("Creating data directory.") - tf.io.gfile.makedirs(self.path_to_data) - logging.info("Downloading license.") - local_license_path = os.path.join( - self.path_to_data, DATA_URL_LICENSE.split("/")[-1]) - if not tf.io.gfile.exists(local_license_path): - urlretrieve(DATA_URL_LICENSE, local_license_path) - logging.info("Downloading annotations.") - local_annotations_path = os.path.join( - self.path_to_data, DATA_URL_ANNOTATIONS.split("/")[-1]) - if not tf.io.gfile.exists(local_annotations_path): - urlretrieve(DATA_URL_ANNOTATIONS, local_annotations_path) - logging.info("Downloading videos.") - local_videos_path = os.path.join( - self.path_to_data, DATA_URL_VIDEOS.split("/")[-1]) - if not tf.io.gfile.exists(local_videos_path): - urlretrieve(DATA_URL_VIDEOS, local_videos_path, progress_hook) - logging.info("Extracting annotations.") - # return video dir and annotation_dir by removing .zip from the path. - annotations_dir = local_annotations_path[:-4] - if not tf.io.gfile.exists(annotations_dir): - with zipfile.ZipFile(local_annotations_path) as annotations_zip: - annotations_zip.extractall(self.path_to_data) - logging.info("Extracting videos.") - video_dir = local_videos_path[:-4] - if not tf.io.gfile.exists(video_dir): - with zipfile.ZipFile(local_videos_path) as videos_zip: - videos_zip.extractall(self.path_to_data) - return annotations_dir, video_dir - - def _run_mediapipe(self, path_to_mediapipe_binary, sequence_example, graph): - """Runs MediaPipe over MediaSequence tf.train.SequenceExamples. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - sequence_example: The SequenceExample with metadata or partial data file. - graph: The path to the graph that extracts data to add to the - SequenceExample. - Returns: - A copy of the input SequenceExample with additional data fields added - by the MediaPipe graph. - Raises: - RuntimeError: if MediaPipe returns an error or fails to run the graph. - """ - if not path_to_mediapipe_binary: - raise ValueError("--path_to_mediapipe_binary must be specified.") - input_fd, input_filename = tempfile.mkstemp() - output_fd, output_filename = tempfile.mkstemp() - cmd = [path_to_mediapipe_binary, - "--calculator_graph_config_file=%s" % graph, - "--input_side_packets=input_sequence_example=%s" % input_filename, - "--output_side_packets=output_sequence_example=%s" % output_filename] - with open(input_filename, "wb") as input_file: - input_file.write(sequence_example.SerializeToString()) - mediapipe_output = subprocess.check_output(cmd) - if b"Failed to run the graph" in mediapipe_output: - raise RuntimeError(mediapipe_output) - with open(output_filename, "rb") as output_file: - output_example = tf.train.SequenceExample() - output_example.ParseFromString(output_file.read()) - os.close(input_fd) - os.remove(input_filename) - os.close(output_fd) - os.remove(output_filename) - return output_example - - -def one_hot_segments(start_indices, end_indices, num_samples): - """Returns a one-hot, float matrix of segments at each timestep. - - All integers in the inclusive range of start_indices and end_indices are used. - This allows start and end timestamps to be mapped to the same index and the - segment will not be omitted. - - Args: - start_indices: a 1d tensor of integer indices for the start of each - segement. - end_indices: a tensor of integer indices for the end of each segment. - Must be the same shape as start_indices. Values should be >= start_indices - but not strictly enforced. - num_samples: the number of rows in the output. Indices should be < - num_samples, but this is not strictly enforced. - Returns: - (segments, indicator) - segments: A [num_samples, num_elements(start_indices)] tensor where in each - column the rows with indices >= start_indices[column] and - <= end_indices[column] are 1.0 and all other values are 0.0. - indicator: a tensor of 1.0 values with shape [num_samples, 1]. If padded - with zeros to align sizes, the indicator marks where segments is valid. - """ - start_indices = tf.convert_to_tensor(start_indices) - end_indices = tf.convert_to_tensor(end_indices) - start_indices.shape.assert_is_compatible_with(end_indices.shape) - start_indices.shape.assert_has_rank(1) - end_indices.shape.assert_has_rank(1) - # create a matrix of the index at each row with a column per segment. - indices = tf.to_int64( - tf.tile( - tf.transpose(tf.expand_dims(tf.range(num_samples), 0)), - [1, tf.shape(start_indices)[0]])) - # switch to one hot encoding of segments (includes start and end indices) - segments = tf.to_float( - tf.logical_and( - tf.greater_equal(indices, start_indices), - tf.less_equal(indices, end_indices))) - # create a tensors of ones everywhere there's an annotation. If padded with - # zeros later, element-wise multiplication of the loss will mask out the - # padding. - indicator = tf.ones(shape=[num_samples, 1], dtype=tf.float32) - return segments, indicator - - -def timepoint_classification_target(segments, segment_classes, num_classes): - """Produces a classification target at each timepoint. - - If no segments are present at a time point, the first class is set to 1.0. - This should be used as a background class unless segments are always present. - - Args: - segments: a [time, num_segments] tensor that is 1.0 at indices within - each segment and 0.0 elsewhere. - segment_classes: a [num_segments] tensor with the class index of each - segment. - num_classes: the number of classes (must be >= max(segment_classes) + 1) - Returns: - a [time, num_classes] tensor. In the final output, more than one - value in a row can be 1.0 if segments overlap. - """ - num_segments = tf.shape(segments)[1] - matrix_of_class_indices = tf.to_int32( - segments * tf.to_float(tf.expand_dims(segment_classes, 0))) - # First column will have one count per zero segment. Correct this to be 0 - # unless no segments are present. - one_hot = tf.reduce_sum(tf.one_hot(matrix_of_class_indices, num_classes), 1) - normalizer = tf.concat([ - tf.ones(shape=[1, 1], dtype=tf.float32) / tf.to_float(num_segments), - tf.ones(shape=[1, num_classes - 1], dtype=tf.float32) - ], 1) - corrected_one_hot = tf.floor(one_hot * normalizer) - return corrected_one_hot - - -def progress_hook(blocks, block_size, total_size): - print("Downloaded %d%% of %d bytes (%d blocks)\r" % ( - blocks * block_size / total_size * 100, total_size, blocks), end="") - - -def bytes23(string): - """Creates a bytes string in either Python 2 or 3.""" - if sys.version_info >= (3, 0): - return bytes(string, "utf8") - else: - return bytes(string) - - -@contextlib.contextmanager -def _close_on_exit(writers): - """Call close on all writers on exit.""" - try: - yield writers - finally: - for writer in writers: - writer.close() - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - Charades(flags.FLAGS.path_to_charades_data).generate_examples( - flags.FLAGS.path_to_mediapipe_binary, - flags.FLAGS.path_to_graph_directory) - -if __name__ == "__main__": - flags.DEFINE_string("path_to_charades_data", - "", - "Path to directory to write data to.") - flags.DEFINE_string("path_to_mediapipe_binary", - "", - "Path to the MediaPipe run_graph_file_io_main binary.") - flags.DEFINE_string("path_to_graph_directory", - "", - "Path to directory containing the graph files.") - app.run(main) diff --git a/mediapipe/examples/desktop/media_sequence/demo_dataset.py b/mediapipe/examples/desktop/media_sequence/demo_dataset.py deleted file mode 100644 index 93ea56e96..000000000 --- a/mediapipe/examples/desktop/media_sequence/demo_dataset.py +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""A demo data set constructed with MediaSequence and MediaPipe. - -This code demonstrates the steps for constructing a data set with MediaSequence. -This code has two functions. First, it can be run as a module to download and -prepare a toy dataset. Second, it can be imported and used to provide a -tf.data.Dataset reading that data from disk via as_dataset(). - -Running as a module prepares the data in three stages via generate_examples(). -First, the actual data files are downloaded. If the download is disrupted, the -incomplete files will need to be removed before running the script again. -Second, the annotations are parsed and reformated into metadata as described in -the MediaSequence documentation. Third, MediaPipe is run to extract subsequences -of frames for subsequent training via _run_mediapipe(). - -The toy data set is classifying a clip as a panning shot of galaxy or nebula -from videos releasued under the [Creative Commons Attribution 4.0 International -license](http://creativecommons.org/licenses/by/4.0/) on the ESA/Hubble site. -(The use of these ESA/Hubble materials does not imply the endorsement by -ESA/Hubble or any ESA/Hubble employee of a commercial product or service.) Each -video is split into 5 or 6 ten-second clips with a label of "galaxy" or "nebula" -and downsampled to 10 frames per second. (The last clip for each test example is -only 6 seconds.) There is one video of each class in each of the training and -testing splits. - -Reading the data as a tf.data.Dataset is accomplished with the following lines: - - demo = DemoDataset("demo_data_path") - dataset = demo.as_dataset("test") - # implement additional processing and batching here - images_and_labels = dataset.make_one_shot_iterator().get_next() - images = images_and_labels["images"] - labels = image_and_labels["labels"] -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import csv -import os -import random -import subprocess -import sys -import tempfile - -from absl import app -from absl import flags -from absl import logging -from six.moves import range -from six.moves import urllib -import tensorflow.compat.v1 as tf - -from mediapipe.util.sequence import media_sequence as ms - -SPLITS = { - "train": - """url,label index,label string,duration,credits -https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1608c.mp4,0,nebula,50,"ESA/Hubble; Music: Johan B. Monell" -https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1712b.mp4,1,galaxy,50,"ESA/Hubble, Digitized Sky Survey, Nick Risinger (skysurvey.org) Music: Johan B Monell" -""", - "test": - """url,label index,label string,duration,credits -https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1301b.m4v,0,nebula,56,"NASA, ESA. Acknowledgement: Josh Lake" -https://cdn.spacetelescope.org/archives/videos/medium_podcast/heic1305b.m4v,1,galaxy,56,"NASA, ESA, Digitized Sky Survey 2. Acknowledgement: A. van der Hoeven" -""" -} -NUM_CLASSES = 2 -NUM_SHARDS = 2 -SECONDS_PER_EXAMPLE = 10 -MICROSECONDS_PER_SECOND = 1000000 -TF_RECORD_PATTERN = "demo_space_dataset_%s_tfrecord" -GRAPHS = ["clipped_images_from_file_at_24fps.pbtxt"] - - -class DemoDataset(object): - """Generates and loads a demo data set.""" - - def __init__(self, path_to_data): - if not path_to_data: - raise ValueError("You must supply the path to the data directory.") - self.path_to_data = path_to_data - - def as_dataset(self, - split, - shuffle=False, - repeat=False, - serialized_prefetch_size=32, - decoded_prefetch_size=32): - """Returns the dataset as a tf.data.Dataset. - - Args: - split: either "train" or "test" - shuffle: if true, shuffles both files and examples. - repeat: if true, repeats the data set forever. - serialized_prefetch_size: the buffer size for reading from disk. - decoded_prefetch_size: the buffer size after decoding. - - Returns: - A tf.data.Dataset object with the following structure: { - "images": uint8 tensor, shape [time, height, width, channels] - "labels": one hot encoded label tensor, shape [2] - "id": a unique string id for each example, shape [] - } - """ - - def parse_fn(sequence_example): - """Parses a clip classification example.""" - context_features = { - ms.get_example_id_key(): - ms.get_example_id_default_parser(), - ms.get_clip_label_index_key(): - ms.get_clip_label_index_default_parser(), - ms.get_clip_label_string_key(): - ms.get_clip_label_string_default_parser() - } - sequence_features = { - ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(), - } - parsed_context, parsed_sequence = tf.io.parse_single_sequence_example( - sequence_example, context_features, sequence_features) - example_id = parsed_context[ms.get_example_id_key()] - classification_target = tf.one_hot( - tf.sparse_tensor_to_dense( - parsed_context[ms.get_clip_label_index_key()]), NUM_CLASSES) - images = tf.map_fn( - tf.image.decode_jpeg, - parsed_sequence[ms.get_image_encoded_key()], - back_prop=False, - dtype=tf.uint8) - return { - "id": example_id, - "labels": classification_target, - "images": images, - } - - if split not in SPLITS: - raise ValueError("split '%s' is unknown." % split) - all_shards = tf.io.gfile.glob( - os.path.join(self.path_to_data, TF_RECORD_PATTERN % split + "-*-of-*")) - if shuffle: - random.shuffle(all_shards) - all_shards_dataset = tf.data.Dataset.from_tensor_slices(all_shards) - cycle_length = min(16, len(all_shards)) - dataset = all_shards_dataset.apply( - tf.data.experimental.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=cycle_length, - block_length=1, - sloppy=True, - buffer_output_elements=serialized_prefetch_size)) - dataset = dataset.prefetch(serialized_prefetch_size) - if shuffle: - dataset = dataset.shuffle(serialized_prefetch_size) - if repeat: - dataset = dataset.repeat() - dataset = dataset.map(parse_fn) - dataset = dataset.prefetch(decoded_prefetch_size) - return dataset - - def generate_examples(self, path_to_mediapipe_binary, - path_to_graph_directory): - """Downloads data and generates sharded TFRecords. - - Downloads the data files, generates metadata, and processes the metadata - with MediaPipe to produce tf.SequenceExamples for training. The resulting - files can be read with as_dataset(). After running this function the - original data files can be deleted. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - path_to_graph_directory: Path to the directory with MediaPipe graphs in - mediapipe/graphs/media_sequence/. - """ - if not path_to_mediapipe_binary: - raise ValueError("You must supply the path to the MediaPipe binary for " - "mediapipe/examples/desktop/demo:media_sequence_demo.") - if not path_to_graph_directory: - raise ValueError( - "You must supply the path to the directory with MediaPipe graphs in " - "mediapipe/graphs/media_sequence/.") - logging.info("Downloading data.") - tf.io.gfile.makedirs(self.path_to_data) - if sys.version_info >= (3, 0): - urlretrieve = urllib.request.urlretrieve - else: - urlretrieve = urllib.request.urlretrieve - for split in SPLITS: - reader = csv.DictReader(SPLITS[split].split("\n")) - all_metadata = [] - for row in reader: - url = row["url"] - basename = url.split("/")[-1] - local_path = os.path.join(self.path_to_data, basename) - if not tf.io.gfile.exists(local_path): - urlretrieve(url, local_path) - - for start_time in range(0, int(row["duration"]), SECONDS_PER_EXAMPLE): - metadata = tf.train.SequenceExample() - ms.set_example_id(bytes23(basename + "_" + str(start_time)), - metadata) - ms.set_clip_data_path(bytes23(local_path), metadata) - ms.set_clip_start_timestamp(start_time * MICROSECONDS_PER_SECOND, - metadata) - ms.set_clip_end_timestamp( - (start_time + SECONDS_PER_EXAMPLE) * MICROSECONDS_PER_SECOND, - metadata) - ms.set_clip_label_index((int(row["label index"]),), metadata) - ms.set_clip_label_string((bytes23(row["label string"]),), - metadata) - all_metadata.append(metadata) - random.seed(47) - random.shuffle(all_metadata) - shard_names = [self._indexed_shard(split, i) for i in range(NUM_SHARDS)] - writers = [tf.io.TFRecordWriter(shard_name) for shard_name in shard_names] - with _close_on_exit(writers) as writers: - for i, seq_ex in enumerate(all_metadata): - for graph in GRAPHS: - graph_path = os.path.join(path_to_graph_directory, graph) - seq_ex = self._run_mediapipe(path_to_mediapipe_binary, seq_ex, - graph_path) - writers[i % len(writers)].write(seq_ex.SerializeToString()) - - def _indexed_shard(self, split, index): - """Constructs a sharded filename.""" - return os.path.join( - self.path_to_data, - TF_RECORD_PATTERN % split + "-%05d-of-%05d" % (index, NUM_SHARDS)) - - def _run_mediapipe(self, path_to_mediapipe_binary, sequence_example, graph): - """Runs MediaPipe over MediaSequence tf.train.SequenceExamples. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - sequence_example: The SequenceExample with metadata or partial data file. - graph: The path to the graph that extracts data to add to the - SequenceExample. - - Returns: - A copy of the input SequenceExample with additional data fields added - by the MediaPipe graph. - Raises: - RuntimeError: if MediaPipe returns an error or fails to run the graph. - """ - if not path_to_mediapipe_binary: - raise ValueError("--path_to_mediapipe_binary must be specified.") - input_fd, input_filename = tempfile.mkstemp() - output_fd, output_filename = tempfile.mkstemp() - cmd = [ - path_to_mediapipe_binary, - "--calculator_graph_config_file=%s" % graph, - "--input_side_packets=input_sequence_example=%s" % input_filename, - "--output_side_packets=output_sequence_example=%s" % output_filename - ] - with open(input_filename, "wb") as input_file: - input_file.write(sequence_example.SerializeToString()) - mediapipe_output = subprocess.check_output(cmd) - if b"Failed to run the graph" in mediapipe_output: - raise RuntimeError(mediapipe_output) - with open(output_filename, "rb") as output_file: - output_example = tf.train.SequenceExample() - output_example.ParseFromString(output_file.read()) - os.close(input_fd) - os.remove(input_filename) - os.close(output_fd) - os.remove(output_filename) - return output_example - - -def bytes23(string): - """Creates a bytes string in either Python 2 or 3.""" - if sys.version_info >= (3, 0): - return bytes(string, "utf8") - else: - return bytes(string) - - -@contextlib.contextmanager -def _close_on_exit(writers): - """Call close on all writers on exit.""" - try: - yield writers - finally: - for writer in writers: - writer.close() - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - DemoDataset(flags.FLAGS.path_to_demo_data).generate_examples( - flags.FLAGS.path_to_mediapipe_binary, flags.FLAGS.path_to_graph_directory) - - -if __name__ == "__main__": - flags.DEFINE_string("path_to_demo_data", "", - "Path to directory to write data to.") - flags.DEFINE_string("path_to_mediapipe_binary", "", - "Path to the MediaPipe run_graph_file_io_main binary.") - flags.DEFINE_string("path_to_graph_directory", "", - "Path to directory containing the graph files.") - app.run(main) diff --git a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py deleted file mode 100644 index eafe18f77..000000000 --- a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py +++ /dev/null @@ -1,478 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -r"""Code to download and parse the Kinetics dataset for TensorFlow models. - -The [Kinetics data set]( -https://deepmind.com/research/open-source/open-source-datasets/kinetics/) -is a data set for human action recognition maintained by DeepMind and Google. -This script downloads the annotations and prepares data from similar annotations -if local video files are available. - -This script does not provide any means of accessing YouTube videos. - -Running this code as a module generates the data set on disk. First, the -required files are downloaded (_download_data) which enables constructing the -label map. Then (in generate_examples), for each split in the data set, the -metadata is generated from the annotations for each example -(_generate_metadata), and MediaPipe is used to fill in the video frames -(_run_mediapipe). This script processes local video files defined in a custom -CSV in a comparable manner to the Kinetics data set for evaluating and -predicting values on your own data. The data set is written to disk as a set of -numbered TFRecord files. - -The custom CSV format must match the Kinetics data set format, with columns -corresponding to [[label_name], video, start, end, split] followed by lines with -those fields. (Label_name is optional.) These field names can be used to -construct the paths to the video files using the Python string formatting -specification and the video_path_format_string flag: - --video_path_format_string="/path/to/video/{video}.mp4" - -Generating the data on disk can take considerable time and disk space. -(Image compression quality is the primary determiner of disk usage. TVL1 flow -determines runtime.) - -Once the data is on disk, reading the data as a tf.data.Dataset is accomplished -with the following lines: - - kinetics = Kinetics("kinetics_data_path") - dataset = kinetics.as_dataset("custom") - # implement additional processing and batching here - images_and_labels = dataset.make_one_shot_iterator().get_next() - images = images_and_labels["images"] - labels = image_and_labels["labels"] - -This data is structured for per-clip action classification where images is -the sequence of images and labels are a one-hot encoded value. See -as_dataset() for more details. - -Note that the number of videos changes in the data set over time, so it will -likely be necessary to change the expected number of examples. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import csv -import os -import random -import subprocess -import sys -import tarfile -import tempfile - -from absl import app -from absl import flags -from absl import logging -from six.moves import range -from six.moves import urllib -from six.moves import zip -import tensorflow.compat.v1 as tf - -from mediapipe.util.sequence import media_sequence as ms - -CITATION = r"""@article{kay2017kinetics, - title={The kinetics human action video dataset}, - author={Kay, Will and Carreira, Joao and Simonyan, Karen and Zhang, Brian and Hillier, Chloe and Vijayanarasimhan, Sudheendra and Viola, Fabio and Green, Tim and Back, Trevor and Natsev, Paul and others}, - journal={arXiv preprint arXiv:1705.06950}, - year={2017}, - url = {https://deepmind.com/research/open-source/kinetics}, -}""" -ANNOTATION_URL = "https://storage.googleapis.com/deepmind-media/Datasets/kinetics700.tar.gz" -SECONDS_TO_MICROSECONDS = 1000000 -GRAPHS = ["tvl1_flow_and_rgb_from_file.pbtxt"] -FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow" -SPLITS = { - "train": { - "shards": 1000, - "examples": 538779 - }, - "validate": { - "shards": 100, - "examples": 34499 - }, - "test": { - "shards": 100, - "examples": 68847 - }, - "custom": { - "csv": None, # Add a CSV for your own data here. - "shards": 1, # Change this number to increase sharding. - "examples": -1 - }, # Negative 1 allows any number of examples. -} -NUM_CLASSES = 700 - - -class Kinetics(object): - """Generates and loads the Kinetics data set.""" - - def __init__(self, path_to_data): - if not path_to_data: - raise ValueError("You must supply the path to the data directory.") - self.path_to_data = path_to_data - - def as_dataset(self, split, shuffle=False, repeat=False, - serialized_prefetch_size=32, decoded_prefetch_size=32, - parse_labels=True): - """Returns Kinetics as a tf.data.Dataset. - - After running this function, calling padded_batch() on the Dataset object - will produce batches of data, but additional preprocessing may be desired. - If using padded_batch, the indicator_matrix output distinguishes valid - from padded frames. - - Args: - split: either "train" or "test" - shuffle: if true, shuffles both files and examples. - repeat: if true, repeats the data set forever. - serialized_prefetch_size: the buffer size for reading from disk. - decoded_prefetch_size: the buffer size after decoding. - parse_labels: if true, also returns the "labels" below. The only - case where this should be false is if the data set was not constructed - with a label map, resulting in this field being missing. - Returns: - A tf.data.Dataset object with the following structure: { - "images": float tensor, shape [time, height, width, channels] - "flow": float tensor, shape [time, height, width, 2] - "num_frames": int32 tensor, shape [], number of frames in the sequence - "labels": float32 tensor, shape [num_classes], one hot encoded. Only - present if parse_labels is true. - """ - logging.info("If you see an error about labels, and you don't supply " - "labels in your CSV, set parse_labels=False") - def parse_fn(sequence_example): - """Parses a Kinetics example.""" - context_features = { - ms.get_example_id_key(): ms.get_example_id_default_parser(), - } - if parse_labels: - context_features[ - ms.get_clip_label_string_key()] = tf.FixedLenFeature((), tf.string) - context_features[ - ms.get_clip_label_index_key()] = tf.FixedLenFeature((), tf.int64) - - sequence_features = { - ms.get_image_encoded_key(): ms.get_image_encoded_default_parser(), - ms.get_forward_flow_encoded_key(): - ms.get_forward_flow_encoded_default_parser(), - } - parsed_context, parsed_sequence = tf.io.parse_single_sequence_example( - sequence_example, context_features, sequence_features) - - images = tf.image.convert_image_dtype( - tf.map_fn(tf.image.decode_jpeg, - parsed_sequence[ms.get_image_encoded_key()], - back_prop=False, - dtype=tf.uint8), tf.float32) - num_frames = tf.shape(images)[0] - - flow = tf.image.convert_image_dtype( - tf.map_fn(tf.image.decode_jpeg, - parsed_sequence[ms.get_forward_flow_encoded_key()], - back_prop=False, - dtype=tf.uint8), tf.float32) - # The flow is quantized for storage in JPEGs by the FlowToImageCalculator. - # The quantization needs to be inverted. - flow = (flow[:, :, :, :2] - 0.5) * 2 * 20. - - output_dict = { - "images": images, - "flow": flow, - "num_frames": num_frames, - } - if parse_labels: - target = tf.one_hot(parsed_context[ms.get_clip_label_index_key()], 700) - output_dict["labels"] = target - return output_dict - - if split not in SPLITS: - raise ValueError("Split %s not in %s" % split, str(list(SPLITS.keys()))) - all_shards = tf.io.gfile.glob( - os.path.join(self.path_to_data, FILEPATTERN % split + "-*-of-*")) - random.shuffle(all_shards) - all_shards_dataset = tf.data.Dataset.from_tensor_slices(all_shards) - cycle_length = min(16, len(all_shards)) - dataset = all_shards_dataset.apply( - tf.contrib.data.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=cycle_length, - block_length=1, sloppy=True, - buffer_output_elements=serialized_prefetch_size)) - dataset = dataset.prefetch(serialized_prefetch_size) - if shuffle: - dataset = dataset.shuffle(serialized_prefetch_size) - if repeat: - dataset = dataset.repeat() - dataset = dataset.map(parse_fn) - dataset = dataset.prefetch(decoded_prefetch_size) - return dataset - - def generate_examples(self, path_to_mediapipe_binary, - path_to_graph_directory, - only_generate_metadata=False, - splits_to_process="train,val,test", - video_path_format_string=None, - download_labels_for_map=True): - """Downloads data and generates sharded TFRecords. - - Downloads the data files, generates metadata, and processes the metadata - with MediaPipe to produce tf.SequenceExamples for training. The resulting - files can be read with as_dataset(). After running this function the - original data files can be deleted. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - path_to_graph_directory: Path to the directory with MediaPipe graphs in - mediapipe/graphs/media_sequence/. - only_generate_metadata: If true, do not run mediapipe and write the - metadata to disk instead. - splits_to_process: csv string of which splits to process. Allows providing - a custom CSV with the CSV flag. The original data is still downloaded - to generate the label_map. - video_path_format_string: The format string for the path to local files. - download_labels_for_map: If true, download the annotations to create the - label map. - """ - if not path_to_mediapipe_binary: - raise ValueError( - "You must supply the path to the MediaPipe binary for " - "mediapipe/examples/desktop/demo:media_sequence_demo.") - if not path_to_graph_directory: - raise ValueError( - "You must supply the path to the directory with MediaPipe graphs in " - "mediapipe/graphs/media_sequence/.") - logging.info("Downloading data.") - download_output = self._download_data(download_labels_for_map) - for key in splits_to_process.split(","): - logging.info("Generating metadata for split: %s", key) - all_metadata = list(self._generate_metadata( - key, download_output, video_path_format_string)) - logging.info("An example of the metadata: ") - logging.info(all_metadata[0]) - random.seed(47) - random.shuffle(all_metadata) - shards = SPLITS[key]["shards"] - shard_names = [os.path.join( - self.path_to_data, FILEPATTERN % key + "-%05d-of-%05d" % ( - i, shards)) for i in range(shards)] - writers = [tf.io.TFRecordWriter(shard_name) for shard_name in shard_names] - with _close_on_exit(writers) as writers: - for i, seq_ex in enumerate(all_metadata): - if not only_generate_metadata: - print("Processing example %d of %d (%d%%) \r" % ( - i, len(all_metadata), i * 100 / len(all_metadata)), end="") - for graph in GRAPHS: - graph_path = os.path.join(path_to_graph_directory, graph) - seq_ex = self._run_mediapipe( - path_to_mediapipe_binary, seq_ex, graph_path) - writers[i % len(writers)].write(seq_ex.SerializeToString()) - logging.info("Data extraction complete.") - - def _generate_metadata(self, key, download_output, - video_path_format_string=None): - """For each row in the annotation CSV, generates the corresponding metadata. - - Args: - key: which split to process. - download_output: the tuple output of _download_data containing - - annotations_files: dict of keys to CSV annotation paths. - - label_map: dict mapping from label strings to numeric indices. - video_path_format_string: The format string for the path to local files. - Yields: - Each tf.SequenceExample of metadata, ready to pass to MediaPipe. - """ - annotations_files, label_map = download_output - with open(annotations_files[key], "r") as annotations: - reader = csv.reader(annotations) - for i, csv_row in enumerate(reader): - if i == 0: # the first row is the header - continue - # rename the row with a constitent set of names. - if len(csv_row) == 5: - row = dict( - list( - zip(["label_name", "video", "start", "end", "split"], - csv_row))) - else: - row = dict(list(zip(["video", "start", "end", "split"], csv_row))) - metadata = tf.train.SequenceExample() - ms.set_example_id(bytes23(row["video"] + "_" + row["start"]), - metadata) - ms.set_clip_media_id(bytes23(row["video"]), metadata) - ms.set_clip_alternative_media_id(bytes23(row["split"]), metadata) - if video_path_format_string: - filepath = video_path_format_string.format(**row) - ms.set_clip_data_path(bytes23(filepath), metadata) - assert row["start"].isdigit(), "Invalid row: %s" % str(row) - assert row["end"].isdigit(), "Invalid row: %s" % str(row) - if "label_name" in row: - ms.set_clip_label_string([bytes23(row["label_name"])], metadata) - if label_map: - ms.set_clip_label_index([label_map[row["label_name"]]], metadata) - yield metadata - - def _download_data(self, download_labels_for_map): - """Downloads and extracts data if not already available.""" - if sys.version_info >= (3, 0): - urlretrieve = urllib.request.urlretrieve - else: - urlretrieve = urllib.request.urlretrieve - logging.info("Creating data directory.") - tf.io.gfile.makedirs(self.path_to_data) - logging.info("Downloading annotations.") - paths = {} - if download_labels_for_map: - tar_path = os.path.join(self.path_to_data, ANNOTATION_URL.split("/")[-1]) - if not tf.io.gfile.exists(tar_path): - urlretrieve(ANNOTATION_URL, tar_path) - with tarfile.open(tar_path) as annotations_tar: - annotations_tar.extractall(self.path_to_data) - for split in ["train", "test", "validate"]: - csv_path = os.path.join(self.path_to_data, "kinetics700/%s.csv" % split) - if not tf.io.gfile.exists(csv_path): - with tarfile.open(tar_path) as annotations_tar: - annotations_tar.extractall(self.path_to_data) - paths[split] = csv_path - for split, contents in SPLITS.items(): - if "csv" in contents and contents["csv"]: - paths[split] = contents["csv"] - label_map = (self.get_label_map_and_verify_example_counts(paths) if - download_labels_for_map else None) - return paths, label_map - - def _run_mediapipe(self, path_to_mediapipe_binary, sequence_example, graph): - """Runs MediaPipe over MediaSequence tf.train.SequenceExamples. - - Args: - path_to_mediapipe_binary: Path to the compiled binary for the BUILD target - mediapipe/examples/desktop/demo:media_sequence_demo. - sequence_example: The SequenceExample with metadata or partial data file. - graph: The path to the graph that extracts data to add to the - SequenceExample. - Returns: - A copy of the input SequenceExample with additional data fields added - by the MediaPipe graph. - Raises: - RuntimeError: if MediaPipe returns an error or fails to run the graph. - """ - if not path_to_mediapipe_binary: - raise ValueError("--path_to_mediapipe_binary must be specified.") - input_fd, input_filename = tempfile.mkstemp() - output_fd, output_filename = tempfile.mkstemp() - cmd = [path_to_mediapipe_binary, - "--calculator_graph_config_file=%s" % graph, - "--input_side_packets=input_sequence_example=%s" % input_filename, - "--output_side_packets=output_sequence_example=%s" % output_filename] - with open(input_filename, "wb") as input_file: - input_file.write(sequence_example.SerializeToString()) - mediapipe_output = subprocess.check_output(cmd) - if b"Failed to run the graph" in mediapipe_output: - raise RuntimeError(mediapipe_output) - with open(output_filename, "rb") as output_file: - output_example = tf.train.SequenceExample() - output_example.ParseFromString(output_file.read()) - os.close(input_fd) - os.remove(input_filename) - os.close(output_fd) - os.remove(output_filename) - return output_example - - def get_label_map_and_verify_example_counts(self, paths): - """Verify the number of examples and labels have not changed.""" - for name, path in paths.items(): - with open(path, "r") as f: - lines = f.readlines() - # the header adds one line and one "key". - num_examples = len(lines) - 1 - keys = [l.split(",")[0] for l in lines] - label_map = None - if name == "train": - classes = sorted(list(set(keys[1:]))) - num_keys = len(set(keys)) - 1 - assert NUM_CLASSES == num_keys, ( - "Found %d labels for split: %s, should be %d" % ( - num_keys, name, NUM_CLASSES)) - label_map = dict(list(zip(classes, list(range(len(classes)))))) - if SPLITS[name]["examples"] > 0: - assert SPLITS[name]["examples"] == num_examples, ( - "Found %d examples for split: %s, should be %d" % ( - num_examples, name, SPLITS[name]["examples"])) - return label_map - - -def bytes23(string): - """Creates a bytes string in either Python 2 or 3.""" - if sys.version_info >= (3, 0): - return bytes(string, "utf8") - else: - return bytes(string) - - -@contextlib.contextmanager -def _close_on_exit(writers): - """Call close on all writers on exit.""" - try: - yield writers - finally: - for writer in writers: - writer.close() - - -def main(argv): - if len(argv) > 1: - raise app.UsageError("Too many command-line arguments.") - if flags.FLAGS.path_to_custom_csv: - SPLITS["custom"]["csv"] = flags.FLAGS.path_to_custom_csv - Kinetics(flags.FLAGS.path_to_kinetics_data).generate_examples( - flags.FLAGS.path_to_mediapipe_binary, - flags.FLAGS.path_to_graph_directory, - flags.FLAGS.only_generate_metadata, - flags.FLAGS.splits_to_process, - flags.FLAGS.video_path_format_string, - flags.FLAGS.download_labels_for_map) - -if __name__ == "__main__": - flags.DEFINE_string("path_to_kinetics_data", - "", - "Path to directory to write data to.") - flags.DEFINE_string("path_to_mediapipe_binary", - "", - "Path to the MediaPipe run_graph_file_io_main binary.") - flags.DEFINE_string("path_to_graph_directory", - "", - "Path to directory containing the graph files.") - flags.DEFINE_boolean("only_generate_metadata", - False, - "If true, only generate the metadata files.") - flags.DEFINE_boolean("download_labels_for_map", - True, - "If true, download the annotations to construct the " - "label map.") - flags.DEFINE_string("splits_to_process", - "custom", - "Process these splits. Useful for custom data splits.") - flags.DEFINE_string("video_path_format_string", - None, - "The format string for the path to local video files. " - "Uses the Python string.format() syntax with possible " - "arguments of {video}, {start}, {end}, {label_name}, and " - "{split}, corresponding to columns of the data csvs.") - flags.DEFINE_string("path_to_custom_csv", - None, - "If present, processes this CSV as a custom split.") - app.run(main) diff --git a/mediapipe/examples/desktop/media_sequence/read_demo_dataset.py b/mediapipe/examples/desktop/media_sequence/read_demo_dataset.py deleted file mode 100644 index b84d991b5..000000000 --- a/mediapipe/examples/desktop/media_sequence/read_demo_dataset.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Lint as: python3 -"""Example of reading a MediaSequence dataset. -""" - -from absl import app -from absl import flags - -from mediapipe.examples.desktop.media_sequence.demo_dataset import DemoDataset -import tensorflow as tf - -FLAGS = flags.FLAGS - - -def main(argv): - if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') - demo_data_path = '/tmp/demo_data/' - with tf.Graph().as_default(): - d = DemoDataset(demo_data_path) - dataset = d.as_dataset('test') - # implement additional processing and batching here - dataset_output = dataset.make_one_shot_iterator().get_next() - images = dataset_output['images'] - labels = dataset_output['labels'] - - with tf.Session() as sess: - images_, labels_ = sess.run([images, labels]) - print('The shape of images_ is %s' % str(images_.shape)) # pylint: disable=superfluous-parens - print('The shape of labels_ is %s' % str(labels_.shape)) # pylint: disable=superfluous-parens - - -if __name__ == '__main__': - app.run(main) diff --git a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc b/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc deleted file mode 100644 index 06212b013..000000000 --- a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// A simple main function to run a MediaPipe graph. Input side packets are read -// from files provided via the command line and output side packets are written -// to disk. -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "absl/strings/str_split.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/map_util.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); -ABSL_FLAG(std::string, input_side_packets, "", - "Comma-separated list of key=value pairs specifying side packets " - "and corresponding file paths for the CalculatorGraph. The side " - "packets are read from the files and fed to the graph as strings " - "even if they represent doubles, floats, etc."); -ABSL_FLAG(std::string, output_side_packets, "", - "Comma-separated list of key=value pairs specifying the output " - "side packets and paths to write to disk for the " - "CalculatorGraph."); - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - std::map input_side_packets; - std::vector kv_pairs = - absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ','); - for (const std::string& kv_pair : kv_pairs) { - std::vector name_and_value = absl::StrSplit(kv_pair, '='); - RET_CHECK(name_and_value.size() == 2); - RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); - std::string input_side_packet_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - name_and_value[1], &input_side_packet_contents)); - input_side_packets[name_and_value[0]] = - mediapipe::MakePacket(input_side_packet_contents); - } - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); - LOG(INFO) << "Start running the calculator graph."; - MP_RETURN_IF_ERROR(graph.Run()); - LOG(INFO) << "Gathering output side packets."; - kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ','); - for (const std::string& kv_pair : kv_pairs) { - std::vector name_and_value = absl::StrSplit(kv_pair, '='); - RET_CHECK(name_and_value.size() == 2); - absl::StatusOr output_packet = - graph.GetOutputSidePacket(name_and_value[0]); - RET_CHECK(output_packet.ok()) - << "Packet " << name_and_value[0] << " was not available."; - const std::string& serialized_string = - output_packet.value().Get(); - MP_RETURN_IF_ERROR( - mediapipe::file::SetContents(name_and_value[1], serialized_string)); - } - return absl::OkStatus(); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/object_detection/BUILD b/mediapipe/examples/desktop/object_detection/BUILD deleted file mode 100644 index c7860f09a..000000000 --- a/mediapipe/examples/desktop/object_detection/BUILD +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "object_detection_tensorflow", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_tensorflow_calculators", - "@org_tensorflow//tensorflow/core:all_kernels", - "@org_tensorflow//tensorflow/core:direct_session", - ], -) - -cc_binary( - name = "object_detection_tflite", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_tflite_calculators", - ], -) - -cc_binary( - name = "object_detection_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/object_detection:desktop_tflite_calculators", - ], -) diff --git a/mediapipe/examples/desktop/object_detection/test_video.mp4 b/mediapipe/examples/desktop/object_detection/test_video.mp4 deleted file mode 100644 index c706232d3..000000000 Binary files a/mediapipe/examples/desktop/object_detection/test_video.mp4 and /dev/null differ diff --git a/mediapipe/examples/desktop/object_detection_3d/BUILD b/mediapipe/examples/desktop/object_detection_3d/BUILD deleted file mode 100644 index 8a58e1129..000000000 --- a/mediapipe/examples/desktop/object_detection_3d/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -# bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/object_detection_3d:objectron_cpu -# To run 3D object detection for shoes, -# bazel-bin/mediapipe/examples/desktop/object_detection_3d/objectron_cpu \ -# --calculator_graph_config_file=mediapipe/graphs/object_detection_3d/objectron_desktop_cpu.pbtxt \ -# --input_side_packets="input_video_path=,box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_sneakers.tflite,output_video_path=,allowed_labels=Footwear" -# To detect objects from other categories, change box_landmark_model_path and allowed_labels accordingly. -# Chair: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_chair.tflite,allowed_labels=Chair -# Camera: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_camera.tflite,allowed_labels=Camera -# Cup: box_landmark_model_path=mediapipe/modules/objectron/object_detection_3d_cup.tflite,allowed_labels=Mug -cc_binary( - name = "objectron_cpu", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/object_detection_3d:desktop_cpu_calculators", - ], -) diff --git a/mediapipe/examples/desktop/object_tracking/BUILD b/mediapipe/examples/desktop/object_tracking/BUILD deleted file mode 100644 index 8a87c5bbc..000000000 --- a/mediapipe/examples/desktop/object_tracking/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "object_tracking_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/tracking:desktop_calculators", - ], -) diff --git a/mediapipe/examples/desktop/pose_tracking/BUILD b/mediapipe/examples/desktop/pose_tracking/BUILD deleted file mode 100644 index 447e2dfdc..000000000 --- a/mediapipe/examples/desktop/pose_tracking/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "pose_tracking_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/pose_tracking:pose_tracking_cpu_deps", - ], -) - -# Linux only -cc_binary( - name = "pose_tracking_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/pose_tracking:pose_tracking_gpu_deps", - ], -) diff --git a/mediapipe/examples/desktop/selfie_segmentation/BUILD b/mediapipe/examples/desktop/selfie_segmentation/BUILD deleted file mode 100644 index ae93aa94c..000000000 --- a/mediapipe/examples/desktop/selfie_segmentation/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "selfie_segmentation_cpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_cpu_deps", - ], -) - -# Linux only -cc_binary( - name = "selfie_segmentation_gpu", - deps = [ - "//mediapipe/examples/desktop:demo_run_graph_main_gpu", - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu_deps", - ], -) diff --git a/mediapipe/examples/desktop/simple_run_graph_main.cc b/mediapipe/examples/desktop/simple_run_graph_main.cc deleted file mode 100644 index 96d9839a8..000000000 --- a/mediapipe/examples/desktop/simple_run_graph_main.cc +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// A simple main function to run a MediaPipe graph. -#include -#include -#include -#include -#include -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_split.h" -#include "absl/strings/string_view.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/map_util.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/statusor.h" - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); - -ABSL_FLAG(std::string, input_side_packets, "", - "Comma-separated list of key=value pairs specifying side packets " - "for the CalculatorGraph. All values will be treated as the " - "string type even if they represent doubles, floats, etc."); - -// Local file output flags. -// Output stream -ABSL_FLAG(std::string, output_stream, "", - "The output stream to output to the local file in csv format."); -ABSL_FLAG(std::string, output_stream_file, "", - "The name of the local file to output all packets sent to " - "the stream specified with --output_stream. "); -ABSL_FLAG(bool, strip_timestamps, false, - "If true, only the packet contents (without timestamps) will be " - "written into the local file."); -// Output side packets -ABSL_FLAG(std::string, output_side_packets, "", - "A CSV of output side packets to output to local file."); -ABSL_FLAG(std::string, output_side_packets_file, "", - "The name of the local file to output all side packets specified " - "with --output_side_packets. "); - -absl::Status OutputStreamToLocalFile(mediapipe::OutputStreamPoller& poller) { - std::ofstream file; - file.open(absl::GetFlag(FLAGS_output_stream_file)); - mediapipe::Packet packet; - while (poller.Next(&packet)) { - std::string output_data; - if (!absl::GetFlag(FLAGS_strip_timestamps)) { - absl::StrAppend(&output_data, packet.Timestamp().Value(), ","); - } - absl::StrAppend(&output_data, packet.Get(), "\n"); - file << output_data; - } - file.close(); - return absl::OkStatus(); -} - -absl::Status OutputSidePacketsToLocalFile(mediapipe::CalculatorGraph& graph) { - if (!absl::GetFlag(FLAGS_output_side_packets).empty() && - !absl::GetFlag(FLAGS_output_side_packets_file).empty()) { - std::ofstream file; - file.open(absl::GetFlag(FLAGS_output_side_packets_file)); - std::vector side_packet_names = - absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ','); - for (const std::string& side_packet_name : side_packet_names) { - ASSIGN_OR_RETURN(auto status_or_packet, - graph.GetOutputSidePacket(side_packet_name)); - file << absl::StrCat(side_packet_name, ":", - status_or_packet.Get(), "\n"); - } - file.close(); - } else { - RET_CHECK(absl::GetFlag(FLAGS_output_side_packets).empty() && - absl::GetFlag(FLAGS_output_side_packets_file).empty()) - << "--output_side_packets and --output_side_packets_file should be " - "specified in pair."; - } - return absl::OkStatus(); -} - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - std::map input_side_packets; - if (!absl::GetFlag(FLAGS_input_side_packets).empty()) { - std::vector kv_pairs = - absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ','); - for (const std::string& kv_pair : kv_pairs) { - std::vector name_and_value = absl::StrSplit(kv_pair, '='); - RET_CHECK(name_and_value.size() == 2); - RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); - input_side_packets[name_and_value[0]] = - mediapipe::MakePacket(name_and_value[1]); - } - } - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); - if (!absl::GetFlag(FLAGS_output_stream).empty() && - !absl::GetFlag(FLAGS_output_stream_file).empty()) { - ASSIGN_OR_RETURN(auto poller, graph.AddOutputStreamPoller( - absl::GetFlag(FLAGS_output_stream))); - LOG(INFO) << "Start running the calculator graph."; - MP_RETURN_IF_ERROR(graph.StartRun({})); - MP_RETURN_IF_ERROR(OutputStreamToLocalFile(poller)); - } else { - RET_CHECK(absl::GetFlag(FLAGS_output_stream).empty() && - absl::GetFlag(FLAGS_output_stream_file).empty()) - << "--output_stream and --output_stream_file should be specified in " - "pair."; - LOG(INFO) << "Start running the calculator graph."; - MP_RETURN_IF_ERROR(graph.StartRun({})); - } - MP_RETURN_IF_ERROR(graph.WaitUntilDone()); - return OutputSidePacketsToLocalFile(graph); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/template_matching/BUILD b/mediapipe/examples/desktop/template_matching/BUILD deleted file mode 100644 index 6ee07f71b..000000000 --- a/mediapipe/examples/desktop/template_matching/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//mediapipe/examples:__subpackages__"]) - -cc_binary( - name = "template_matching_tflite", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/template_matching:desktop_calculators", - ], -) diff --git a/mediapipe/examples/desktop/youtube8m/BUILD b/mediapipe/examples/desktop/youtube8m/BUILD deleted file mode 100644 index e0e44c4d9..000000000 --- a/mediapipe/examples/desktop/youtube8m/BUILD +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -cc_binary( - name = "extract_yt8m_features", - srcs = ["extract_yt8m_features.cc"], - deps = [ - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/graphs/youtube8m:yt8m_feature_extraction_calculators", - # TODO: Figure out the minimum set of the kernels needed by this example. - "@org_tensorflow//tensorflow/core:all_kernels", - "@org_tensorflow//tensorflow/core:direct_session", - ], -) - -cc_binary( - name = "model_inference", - deps = [ - "//mediapipe/examples/desktop:simple_run_graph_main", - "//mediapipe/graphs/youtube8m:yt8m_inference_calculators_deps", - # TODO: Figure out the minimum set of the kernels needed by this example. - "@org_tensorflow//tensorflow/core:all_kernels", - "@org_tensorflow//tensorflow/core:direct_session", - ], -) diff --git a/mediapipe/examples/desktop/youtube8m/README.md b/mediapipe/examples/desktop/youtube8m/README.md deleted file mode 100644 index 94b36809d..000000000 --- a/mediapipe/examples/desktop/youtube8m/README.md +++ /dev/null @@ -1,157 +0,0 @@ -### Steps to run the YouTube-8M feature extraction graph - -1. Checkout the repository and follow - [the installation instructions](https://github.com/google/mediapipe/blob/master/docs/getting_started/install.md) - to set up MediaPipe. - - ```bash - git clone https://github.com/google/mediapipe.git - cd mediapipe - ``` - -2. Download the PCA and model data. - - ```bash - mkdir /tmp/mediapipe - cd /tmp/mediapipe - curl -O http://data.yt8m.org/pca_matrix_data/inception3_mean_matrix_data.pb - curl -O http://data.yt8m.org/pca_matrix_data/inception3_projection_matrix_data.pb - curl -O http://data.yt8m.org/pca_matrix_data/vggish_mean_matrix_data.pb - curl -O http://data.yt8m.org/pca_matrix_data/vggish_projection_matrix_data.pb - curl -O http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz - tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz - ``` - -3. Get the VGGish frozen graph. - - Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed - with the TensorFlow 1.14+ package installed. - - ```bash - # cd to the root directory of the MediaPipe repo - cd - - - pip3 install tf_slim - python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph - ``` - -4. Generate a MediaSequence metadata from the input video. - - Note: the output file is /tmp/mediapipe/metadata.pb - - ```bash - # change clip_end_time_sec to match the length of your video. - python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \ - --path_to_input_video=/absolute/path/to/the/local/video/file \ - --clip_end_time_sec=120 - ``` - -5. Run the MediaPipe binary to extract the features. - - ```bash - bazel build -c opt --linkopt=-s \ - --define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \ - mediapipe/examples/desktop/youtube8m:extract_yt8m_features - - GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features \ - --calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \ - --input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.pb \ - --output_side_packets=output_sequence_example=/tmp/mediapipe/features.pb - ``` - -6. [Optional] Read the features.pb in Python. - - ``` - import tensorflow as tf - - sequence_example = open('/tmp/mediapipe/features.pb', 'rb').read() - print(tf.train.SequenceExample.FromString(sequence_example)) - ``` - -### Steps to run the YouTube-8M inference graph with the YT8M dataset - -1. Download the YT8M dataset - - For example, download one shard of the training data: - - ```bash - curl http://us.data.yt8m.org/2/frame/train/trainpj.tfrecord --output /tmp/mediapipe/trainpj.tfrecord - ``` - -2. Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local. - - ```bash - curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz http://data.yt8m.org/models/baseline/saved_model.tar.gz - - tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe - ``` - -3. Build and run the inference binary. - - ```bash - bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \ - mediapipe/examples/desktop/youtube8m:model_inference - - GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \ - --calculator_graph_config_file=mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt \ - --input_side_packets=tfrecord_path=/tmp/mediapipe/trainpj.tfrecord,record_index=0,desired_segment_size=5 \ - --output_stream=annotation_summary \ - --output_stream_file=/tmp/summary \ - --output_side_packets=yt8m_id \ - --output_side_packets_file=/tmp/yt8m_id - ``` - -### Steps to run the YouTube-8M model inference graph with Web Interface - -1. Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local. - - - ```bash - curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz http://data.yt8m.org/models/baseline/saved_model.tar.gz - - tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe - ``` - -2. Build the inference binary. - - ```bash - bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \ - mediapipe/examples/desktop/youtube8m:model_inference - ``` - -3. Run the python web server. - - Note: pip3 install absl-py - - ```bash - python mediapipe/examples/desktop/youtube8m/viewer/server.py --root `pwd` - ``` - - Navigate to localhost:8008 in a web browser. - -### Steps to run the YouTube-8M model inference graph with a local video - -1. Make sure you have the features.pb from the feature extraction pipeline. - -2. Copy the baseline model [(model card)](https://drive.google.com/file/d/1xTCi9-Nm9dt2KIk8WR0dDFrIssWawyXy/view) to local. - - ```bash - curl -o /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz http://data.yt8m.org/models/baseline/saved_model.tar.gz - - tar -xvf /tmp/mediapipe/yt8m_baseline_saved_model.tar.gz -C /tmp/mediapipe - ``` - -3. Build and run the inference binary. - - ```bash - bazel build -c opt --define='MEDIAPIPE_DISABLE_GPU=1' --linkopt=-s \ - mediapipe/examples/desktop/youtube8m:model_inference - - # segment_size is the number of seconds window of frames. - # overlap is the number of seconds adjacent segments share. - GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference \ - --calculator_graph_config_file=mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt \ - --input_side_packets=input_sequence_example_path=/tmp/mediapipe/features.pb,input_video_path=/absolute/path/to/the/local/video/file,output_video_path=/tmp/mediapipe/annotated_video.mp4,segment_size=5,overlap=4 - ``` - -4. View the annotated video. diff --git a/mediapipe/examples/desktop/youtube8m/__init__.py b/mediapipe/examples/desktop/youtube8m/__init__.py deleted file mode 100644 index 6db73bc52..000000000 --- a/mediapipe/examples/desktop/youtube8m/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Copyright 2019 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc b/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc deleted file mode 100644 index 9030e9255..000000000 --- a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// A simple main function to run a MediaPipe graph. Input side packets are read -// from files provided via the command line and output side packets are written -// to disk. -#include - -#include "absl/flags/flag.h" -#include "absl/flags/parse.h" -#include "absl/strings/str_split.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/map_util.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" - -ABSL_FLAG(std::string, calculator_graph_config_file, "", - "Name of file containing text format CalculatorGraphConfig proto."); -ABSL_FLAG(std::string, input_side_packets, "", - "Comma-separated list of key=value pairs specifying side packets " - "and corresponding file paths for the CalculatorGraph. The side " - "packets are read from the files and fed to the graph as strings " - "even if they represent doubles, floats, etc."); -ABSL_FLAG(std::string, output_side_packets, "", - "Comma-separated list of key=value pairs specifying the output " - "side packets and paths to write to disk for the " - "CalculatorGraph."); - -absl::Status RunMPPGraph() { - std::string calculator_graph_config_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - absl::GetFlag(FLAGS_calculator_graph_config_file), - &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; - mediapipe::CalculatorGraphConfig config = - mediapipe::ParseTextProtoOrDie( - calculator_graph_config_contents); - std::map input_side_packets; - std::vector kv_pairs = - absl::StrSplit(absl::GetFlag(FLAGS_input_side_packets), ','); - for (const std::string& kv_pair : kv_pairs) { - std::vector name_and_value = absl::StrSplit(kv_pair, '='); - RET_CHECK(name_and_value.size() == 2); - RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); - std::string input_side_packet_contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - name_and_value[1], &input_side_packet_contents)); - input_side_packets[name_and_value[0]] = - mediapipe::MakePacket(input_side_packet_contents); - } - - mediapipe::MatrixData inc3_pca_mean_matrix_data, - inc3_pca_projection_matrix_data, vggish_pca_mean_matrix_data, - vggish_pca_projection_matrix_data; - mediapipe::Matrix inc3_pca_mean_matrix, inc3_pca_projection_matrix, - vggish_pca_mean_matrix, vggish_pca_projection_matrix; - - std::string content; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - "/tmp/mediapipe/inception3_mean_matrix_data.pb", &content)); - inc3_pca_mean_matrix_data.ParseFromString(content); - mediapipe::MatrixFromMatrixDataProto(inc3_pca_mean_matrix_data, - &inc3_pca_mean_matrix); - input_side_packets["inception3_pca_mean_matrix"] = - mediapipe::MakePacket(inc3_pca_mean_matrix); - - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - "/tmp/mediapipe/inception3_projection_matrix_data.pb", &content)); - inc3_pca_projection_matrix_data.ParseFromString(content); - mediapipe::MatrixFromMatrixDataProto(inc3_pca_projection_matrix_data, - &inc3_pca_projection_matrix); - input_side_packets["inception3_pca_projection_matrix"] = - mediapipe::MakePacket(inc3_pca_projection_matrix); - - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - "/tmp/mediapipe/vggish_mean_matrix_data.pb", &content)); - vggish_pca_mean_matrix_data.ParseFromString(content); - mediapipe::MatrixFromMatrixDataProto(vggish_pca_mean_matrix_data, - &vggish_pca_mean_matrix); - input_side_packets["vggish_pca_mean_matrix"] = - mediapipe::MakePacket(vggish_pca_mean_matrix); - - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - "/tmp/mediapipe/vggish_projection_matrix_data.pb", &content)); - vggish_pca_projection_matrix_data.ParseFromString(content); - mediapipe::MatrixFromMatrixDataProto(vggish_pca_projection_matrix_data, - &vggish_pca_projection_matrix); - input_side_packets["vggish_pca_projection_matrix"] = - mediapipe::MakePacket(vggish_pca_projection_matrix); - - LOG(INFO) << "Initialize the calculator graph."; - mediapipe::CalculatorGraph graph; - MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); - LOG(INFO) << "Start running the calculator graph."; - MP_RETURN_IF_ERROR(graph.Run()); - LOG(INFO) << "Gathering output side packets."; - kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ','); - for (const std::string& kv_pair : kv_pairs) { - std::vector name_and_value = absl::StrSplit(kv_pair, '='); - RET_CHECK(name_and_value.size() == 2); - absl::StatusOr output_packet = - graph.GetOutputSidePacket(name_and_value[0]); - RET_CHECK(output_packet.ok()) - << "Packet " << name_and_value[0] << " was not available."; - const std::string& serialized_string = - output_packet.value().Get(); - MP_RETURN_IF_ERROR( - mediapipe::file::SetContents(name_and_value[1], serialized_string)); - } - return absl::OkStatus(); -} - -int main(int argc, char** argv) { - google::InitGoogleLogging(argv[0]); - absl::ParseCommandLine(argc, argv); - absl::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); - return EXIT_FAILURE; - } else { - LOG(INFO) << "Success!"; - } - return EXIT_SUCCESS; -} diff --git a/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py b/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py deleted file mode 100644 index 205834cc8..000000000 --- a/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generate a MediaSequence metadata for MediaPipe input.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys - -from absl import app -from absl import flags -import six -import tensorflow.compat.v1 as tf -from mediapipe.util.sequence import media_sequence as ms - -FLAGS = flags.FLAGS -SECONDS_TO_MICROSECONDS = 1000000 - - -def bytes23(string): - """Creates a bytes string in either Python 2 or 3.""" - if sys.version_info >= (3, 0): - return bytes(string, 'utf8') - else: - return bytes(string) - - -def main(argv): - if len(argv) > 3: - raise app.UsageError('Too many command-line arguments.') - - if not flags.FLAGS.path_to_input_video: - raise ValueError('You must specify the path to the input video.') - if not flags.FLAGS.clip_end_time_sec: - raise ValueError('You must specify the clip end timestamp in seconds.') - if flags.FLAGS.clip_start_time_sec >= flags.FLAGS.clip_end_time_sec: - raise ValueError( - 'The clip start time must be greater than the clip end time.') - metadata = tf.train.SequenceExample() - ms.set_clip_data_path(bytes23(flags.FLAGS.path_to_input_video), metadata) - ms.set_clip_start_timestamp( - flags.FLAGS.clip_start_time_sec * SECONDS_TO_MICROSECONDS, metadata) - ms.set_clip_end_timestamp( - flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata) - with open('/tmp/mediapipe/metadata.pb', 'wb') as writer: - writer.write(six.ensure_binary(metadata.SerializeToString())) - - -if __name__ == '__main__': - flags.DEFINE_string('path_to_input_video', '', 'Path to the input video.') - flags.DEFINE_integer('clip_start_time_sec', 0, - 'Clip start timestamp in seconds') - flags.DEFINE_integer('clip_end_time_sec', 10, 'Clip end timestamp in seconds') - app.run(main) diff --git a/mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py b/mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py deleted file mode 100644 index 786e2a6e7..000000000 --- a/mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Code to clone the github repository, download the checkpoint and generate the frozen graph. - -The frozen VGGish checkpoint will be saved to `/tmp/mediapipe/vggish_new.pb`. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys - -from absl import app -import tensorflow.compat.v1 as tf -from tensorflow.python.tools import freeze_graph - -BASE_DIR = '/tmp/mediapipe/' - - -def create_vggish_frozen_graph(): - """Create the VGGish frozen graph.""" - os.system('git clone https://github.com/tensorflow/models.git') - sys.path.append('models/research/audioset/vggish/') - - import vggish_slim - os.system('curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt') - ckpt_path = 'vggish_model.ckpt' - - with tf.Graph().as_default(), tf.Session() as sess: - vggish_slim.define_vggish_slim(training=False) - vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path) - - saver = tf.train.Saver(tf.all_variables()) - - freeze_graph.freeze_graph_with_def_protos( - sess.graph_def, - saver.as_saver_def(), - ckpt_path, - 'vggish/fc2/BiasAdd', - restore_op_name=None, - filename_tensor_name=None, - output_graph='/tmp/mediapipe/vggish_new.pb', - clear_devices=True, - initializer_nodes=None) - os.system('rm -rf models/') - os.system('rm %s' % ckpt_path) - - -def main(argv): - if len(argv) > 1: - raise app.UsageError('Too many command-line arguments.') - - create_vggish_frozen_graph() - - -if __name__ == '__main__': - app.run(main) diff --git a/mediapipe/examples/desktop/youtube8m/viewer/server.py b/mediapipe/examples/desktop/youtube8m/viewer/server.py deleted file mode 100644 index febaad53d..000000000 --- a/mediapipe/examples/desktop/youtube8m/viewer/server.py +++ /dev/null @@ -1,262 +0,0 @@ -"""Server for YouTube8M Model Inference Demo. - -Serves up both the static files for the website and provides a service that -fetches the video id and timestamp based labels for a video analyzed in a -tfrecord files. - -""" -from __future__ import print_function -import json -import os -import re -import socket -import subprocess -import sys - -from absl import app -from absl import flags -import http.client -import http.server -from six.moves.urllib import parse - -FLAGS = flags.FLAGS -flags.DEFINE_bool("show_label_at_center", False, - "Show labels at the center of the segment.") -flags.DEFINE_integer("port", 8008, "Port that the API is served over.") -flags.DEFINE_string("tmp_dir", "/tmp/mediapipe", - "Temporary asset storage location.") -flags.DEFINE_string("root", "", "MediaPipe root directory.") -# binary, pbtxt, label_map paths are relative to 'root' path -flags.DEFINE_string( - "binary", - "bazel-bin/mediapipe/examples/desktop/youtube8m/model_inference", - "Inference binary location.") -flags.DEFINE_string( - "pbtxt", - "mediapipe/graphs/youtube8m/yt8m_dataset_model_inference.pbtxt", - "Default pbtxt graph file.") -flags.DEFINE_string("label_map", "mediapipe/graphs/youtube8m/label_map.txt", - "Default label map text file.") - - -class HTTPServerV6(http.server.HTTPServer): - address_family = socket.AF_INET6 - - -class Youtube8MRequestHandler(http.server.SimpleHTTPRequestHandler): - """Static file server with /healthz support.""" - - def do_GET(self): - if self.path.startswith("/healthz"): - self.send_response(200) - self.send_header("Content-type", "text/plain") - self.send_header("Content-length", 2) - self.end_headers() - self.wfile.write("ok") - if self.path.startswith("/video"): - parsed_params = parse.urlparse(self.path) - url_params = parse.parse_qs(parsed_params.query) - - tfrecord_path = "" - segment_size = 5 - - print(url_params) - if "file" in url_params: - tfrecord_path = url_params["file"][0] - if "segments" in url_params: - segment_size = int(url_params["segments"][0]) - - self.fetch(tfrecord_path, segment_size) - - else: - if self.path == "/": - self.path = "/index.html" - # Default to serve up a local file - self.path = "/static" + self.path - http.server.SimpleHTTPRequestHandler.do_GET(self) - - def report_error(self, msg): - """Simplifies sending out a string as a 500 http response.""" - self.send_response(500) - self.send_header("Content-type", "text/plain") - self.end_headers() - if sys.version_info[0] < 3: - self.wfile.write(str(msg).encode("utf-8")) - else: - self.wfile.write(bytes(msg, "utf-8")) - - def report_missing_files(self, files): - """Sends out 500 response with missing files.""" - accumulate = "" - for file_path in files: - if not os.path.exists(file_path): - accumulate = "%s '%s'" % (accumulate, file_path) - - if accumulate: - self.report_error("Could not find:%s" % accumulate) - return True - - return False - - def fetch(self, path, segment_size): - """Returns the video id and labels for a tfrecord at a provided index.""" - - print("Received request. File=", path, "Segment Size =", segment_size) - - if (self.report_missing_files([ - "%s/%s" % (FLAGS.root, FLAGS.pbtxt), - "%s/%s" % (FLAGS.root, FLAGS.binary), - "%s/%s" % (FLAGS.root, FLAGS.label_map) - ])): - return - - # Parse the youtube video id off the end of the link or as a standalone id. - filename_match = re.match( - "(?:.*youtube.*v=)?([a-zA-Z-0-9_]{2})([a-zA-Z-0-9_]+)", path) - tfrecord_url = filename_match.expand(r"data.yt8m.org/2/j/r/\1/\1\2.js") - - print("Trying to get tfrecord via", tfrecord_url) - - connection = http.client.HTTPConnection("data.yt8m.org") - connection.request("GET", tfrecord_url) - response = connection.getresponse() - - response_object = json.loads(response.read()) - filename = response_object["filename_raw"] - index = response_object["index"] - - print("TFRecord discovered: ", filename, ", index", index) - - output_file = r"%s/%s" % (FLAGS.tmp_dir, filename) - tfrecord_url = r"http://us.data.yt8m.org/2/frame/train/%s" % filename - - connection = http.client.HTTPConnection("us.data.yt8m.org") - connection.request("HEAD", - filename_match.expand(r"/2/frame/train/%s" % filename)) - response = connection.getresponse() - if response.getheader("Content-Type") != "application/octet-stream": - self.report_error("Filename '%s' is invalid." % path) - - print(output_file, "exists on yt8m.org. Did we fetch this before?") - - if not os.path.exists(output_file): - print(output_file, "doesn't exist locally, download it now.") - return_code = subprocess.call( - ["curl", "--output", output_file, tfrecord_url], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if return_code: - self.report_error("Could not retrieve contents from %s" % tfrecord_url) - return - else: - print(output_file, "exist locally, reuse it.") - - print("Run the graph...") - process = subprocess.Popen([ - "%s/%s" % (FLAGS.root, FLAGS.binary), - "--calculator_graph_config_file=%s/%s" % (FLAGS.root, FLAGS.pbtxt), - "--input_side_packets=tfrecord_path=%s" % output_file + - ",record_index=%d" % index + ",desired_segment_size=%d" % segment_size, - "--output_stream=annotation_summary", - "--output_stream_file=%s/labels" % FLAGS.tmp_dir, - "--output_side_packets=yt8m_id", - "--output_side_packets_file=%s/yt8m_id" % FLAGS.tmp_dir - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout_str, stderr_str = process.communicate() - process.wait() - - if stderr_str and "success" not in str(stderr_str).lower(): - self.report_error("Error executing server binary: \n%s" % stderr_str) - return - - f = open("%s/yt8m_id" % FLAGS.tmp_dir, "r") - contents = f.read() - print("yt8m_id is", contents[-5:-1]) - - curl_arg = "data.yt8m.org/2/j/i/%s/%s.js" % (contents[-5:-3], - contents[-5:-1]) - print("Grab labels from", curl_arg) - process = subprocess.Popen(["curl", curl_arg], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout = process.communicate() - process.wait() - - stdout_str = stdout[0].decode("utf-8") - - match = re.match(""".+"([^"]+)"[^"]+""", stdout_str) - final_results = { - "video_id": match.group(1), - "link": "https://www.youtube.com/watch?v=%s" % match.group(1), - "entries": [] - } - f = open("%s/labels" % FLAGS.tmp_dir, "r") - lines = f.readlines() - show_at_center = FLAGS.show_label_at_center - - print("%s/labels" % FLAGS.tmp_dir, "holds", len(lines), "entries") - for line in lines: - entry = {"labels": []} - final_results["entries"].append(entry) - first = True - for column in line.split(","): - if first: - subtract = segment_size / 2.0 if show_at_center else 0.0 - entry["time"] = float(int(column)) / 1000000.0 - subtract - first = False - else: - label_score = re.match("(.+):([0-9.]+).*", column) - if label_score: - score = float(label_score.group(2)) - entry["labels"].append({ - "label": label_score.group(1), - "score": score - }) - else: - print("empty score") - - response_json = json.dumps(final_results, indent=2, separators=(",", ": ")) - self.send_response(200) - self.send_header("Content-type", "application/json") - self.end_headers() - if sys.version_info[0] < 3: - self.wfile.write(str(response_json).encode("utf-8")) - else: - self.wfile.write(bytes(response_json, "utf-8")) - - -def update_pbtxt(): - """Update graph.pbtxt to use full path to label_map.txt.""" - edited_line = "" - lines = [] - with open("%s/%s" % (FLAGS.root, FLAGS.pbtxt), "r") as f: - lines = f.readlines() - for line in lines: - if "label_map_path" in line: - kv = line.split(":") - edited_line = kv[0] + (": \"%s/%s\"\n" % (FLAGS.root, FLAGS.label_map)) - with open("%s/%s" % (FLAGS.root, FLAGS.pbtxt), "w") as f: - for line in lines: - if "label_map_path" in line: - f.write(edited_line) - else: - f.write(line) - - -def main(unused_args): - dname = os.path.dirname(os.path.abspath(__file__)) - os.chdir(dname) - if not FLAGS.root: - print("Must specify MediaPipe root directory: --root `pwd`") - return - update_pbtxt() - port = FLAGS.port - print("Listening on port %s" % port) # pylint: disable=superfluous-parens - server = HTTPServerV6(("::", int(port)), Youtube8MRequestHandler) - server.serve_forever() - - -if __name__ == "__main__": - app.run(main) diff --git a/mediapipe/examples/desktop/youtube8m/viewer/static/index.html b/mediapipe/examples/desktop/youtube8m/viewer/static/index.html deleted file mode 100644 index 400aa0af0..000000000 --- a/mediapipe/examples/desktop/youtube8m/viewer/static/index.html +++ /dev/null @@ -1,96 +0,0 @@ - - - - MediaPipe: YouTube8M Model Inference Demo - - - - - - - - -

-

- MediaPipe: YouTube8M Model Inference Demo -

-
-
-
-
-
- -
-
- -
-
-
-
- -
-
- - - - e.g., Both "https://youtube.com/watch?v=huGVGe3Afng" or "huGVGe3Afng" will work. - -
-
- - -
- - -
-
- -
-
-
- - -
-
- Labels -
- -
-
-
-
-
- - - - - - - - diff --git a/mediapipe/examples/desktop/youtube8m/viewer/static/main.js b/mediapipe/examples/desktop/youtube8m/viewer/static/main.js deleted file mode 100644 index ad66e67ea..000000000 --- a/mediapipe/examples/desktop/youtube8m/viewer/static/main.js +++ /dev/null @@ -1,217 +0,0 @@ -/** - * @license - * Copyright 2019 The MediaPipe Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const STATE_PLAYER=0; -const STATE_COVER=1; -const STATE_SPINNER=2; - -/** -* Looks up the value of a url parameter. -* -* @param {string} param The name of the parameter. -* @return {?string} The parameter value or null if there is no such parameter. -*/ -var getUrlParameter = function(param) { - const url = decodeURIComponent(window.location.search.substring(1)); - const url_parts = url.split('&'); - for (var i = 0; i < url_parts.length; i++) { - const param_name = url_parts[i].split(/=(.*)/); - if (param_name[0] === param) { - return param_name[1] === undefined ? null : param_name[1]; - } - } -}; - -/** -* Sets the fields in the form to match the values of the URL parameters. -*/ -const updateFormFromURL = function() { - const form_elements = document.getElementById('form').elements; - const url = decodeURIComponent(window.location.search.substring(1)); - const url_parts = url.split('&'); - for (var i = 0; i < url_parts.length; i++) { - const p = url_parts[i].split(/=(.*)/); - if (p.length >= 2) { - if (form_elements[p[0]]) { - form_elements[p[0]].value = decodeURIComponent(p[1]); - } - } - } -}; - -let player = null; -let intervalID = undefined; -let entries = []; - -/** - * Constructs the embedded YouTube player. - */ -window.onYouTubeIframeAPIReady = () => { - player = new YT.Player('ytplayer', { - events: { - 'onReady': onPlayerReady, - 'onStateChange': onStateChange - } - }); -}; - - -/** - * Listens for YouTube video events. When video is playing, periodically checks - * the time signature and updates the feedback with labels. When video stops, - * shuts off interval timer to save cycles. - * @param {!Event} event YouTube API Event. - */ -function onStateChange(event) { - if (event.data === 1) { - // Youtube switched to playing. - intervalID = setInterval(function(){ - const currentTime = player.getCurrentTime(); - let winner = undefined; - let first = undefined; - for (entry of entries) { - if (!first) { - first = entry.labels; - } - if (entry.time < currentTime) { - winner = entry.labels; - } else { - break; - } - } - if (!winner) { - winner = first; - } - const threshold = - document.getElementById('form').elements['threshold'].value; - let message = ""; - for (var label of winner) { - if (label.score >= threshold) { - message = `${message}${label.label} (score: ${label.score})\n`; - } - } - $("textarea#feedback").val(message); - }); - } else { - if (intervalID) { - clearInterval(intervalID); - } - } -} - -/** - * Turns elements of the player on and off to reflect the state of the "app". - * @param {number} state One of STATE_COVER | STATE_SPINNER | STATE_PLAYER. - */ -function showState(state) { - switch(state) { - case STATE_COVER: - $('#cover').show(); - $('#spinner').hide(); - $('#ytplayer').hide(); - break; - case STATE_SPINNER: - $('#cover').hide(); - $('#spinner').show(); - $('#ytplayer').hide(); - break; - case STATE_PLAYER: - default: - $('#cover').hide(); - $('#spinner').hide(); - $('#ytplayer').show(); - break; - } -} - -/** - * Hide error field and clear its message. - */ -function hideError() { - $('#error_msg').css("visibility", "hidden").text(''); -} - -/** - * Set the error to visible and set its message. - * @param {string} msg Error message as a string. - */ -function showError(msg) { - $('#error_msg').css("visibility", "visible").text(msg); -} - -/** - * Privides numeric feedback for the slider. - */ -function connectSlider() { - $('#threshold_label').text( - `Score Threshold (${$('#threshold')[0].value})`); - $('#threshold').on('input', () => { - $('#threshold_label').text( - `Score Threshold (${$('#threshold')[0].value})`); - }); - $('#segments_label').text( - `Segment Size (${$('#segments')[0].value})`); - $('#segments').on('input', () => { - $('#segments_label').text( - `Segment Size (${$('#segments')[0].value})`); - }); -} - -/** - * Retrieve video information from backend. - * @param {string} filePath name of a tfrecord file. - * @param {number} segments desired number of segments (1-300) - */ -function fetchVideo(filePath, segments) { - const url = "/video?file=" + filePath + "&segments=" + segments; - $.ajax({ - url: url, - success: function(result) { - const videoId = result["video_id"]; - player.loadVideoById(videoId); - entries = result['entries']; - showState(STATE_PLAYER); - }, - error: (err) => { - showState(STATE_COVER); - console.log(err); - showError(err.responseText); - }, - datatype: "json" - }); -} - -/** - * Called when the embedded YouTube player has finished loading. It loads the - * requested video into the player and calls the golden6_viewer API to retrieve - * the frame-level data for that video. - */ -function onPlayerReady() { - const filePath = getUrlParameter('file') || ""; - const segments = parseInt(getUrlParameter('segments')) || 0; - - updateFormFromURL(); - hideError(); - connectSlider(); - - if (!filePath) { - return; - } - - showState(STATE_SPINNER); - fetchVideo(filePath, segments); -} diff --git a/mediapipe/examples/ios/BUILD b/mediapipe/examples/ios/BUILD deleted file mode 100644 index fd611a615..000000000 --- a/mediapipe/examples/ios/BUILD +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -alias( - name = "provisioning_profile", - actual = "//mediapipe:provisioning_profile.mobileprovision", -) diff --git a/mediapipe/examples/ios/README.md b/mediapipe/examples/ios/README.md deleted file mode 100644 index 82813b566..000000000 --- a/mediapipe/examples/ios/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains MediaPipe example applications for iOS. Please see [Solutions](https://solutions.mediapipe.dev)for details. diff --git a/mediapipe/examples/ios/bundle_id.bzl b/mediapipe/examples/ios/bundle_id.bzl deleted file mode 100644 index 4866b07c6..000000000 --- a/mediapipe/examples/ios/bundle_id.bzl +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Configuration helper for iOS app bundle ids and provisioning profiles. -""" - -BUNDLE_ID_PREFIX = "*SEE_IOS_INSTRUCTIONS*.mediapipe.examples" - -# Look for a provisioning profile in the example's directory first, -# otherwise look for a common one. -def example_provisioning(): - local_profile = native.glob(["provisioning_profile.mobileprovision"]) - if local_profile: - return local_profile[0] - return "//mediapipe/examples/ios:provisioning_profile" diff --git a/mediapipe/examples/ios/common/AppDelegate.h b/mediapipe/examples/ios/common/AppDelegate.h deleted file mode 100644 index 6b0377ef2..000000000 --- a/mediapipe/examples/ios/common/AppDelegate.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -@interface AppDelegate : UIResponder - -@property(strong, nonatomic) UIWindow *window; - -@end diff --git a/mediapipe/examples/ios/common/AppDelegate.mm b/mediapipe/examples/ios/common/AppDelegate.mm deleted file mode 100644 index 1746c2267..000000000 --- a/mediapipe/examples/ios/common/AppDelegate.mm +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "AppDelegate.h" - -#import "mediapipe/examples/ios/common/CommonViewController.h" - -@interface AppDelegate () - -@end - -@implementation AppDelegate - -- (BOOL)application:(UIApplication *)application - didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { - // Override point for customization after application launch. - return YES; -} - -- (void)applicationWillResignActive:(UIApplication *)application { - // Sent when the application is about to move from active to inactive state. This can occur for - // certain types of temporary interruptions (such as an incoming phone call or SMS message) or - // when the user quits the application and it begins the transition to the background state. Use - // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering - // callbacks. Games should use this method to pause the game. -} - -- (void)applicationDidEnterBackground:(UIApplication *)application { - // Use this method to release shared resources, save user data, invalidate timers, and store - // enough application state information to restore your application to its current state in case - // it is terminated later. If your application supports background execution, this method is - // called instead of applicationWillTerminate: when the user quits. -} - -- (void)applicationWillEnterForeground:(UIApplication *)application { - // Called as part of the transition from the background to the active state; here you can undo - // many of the changes made on entering the background. -} - -- (void)applicationDidBecomeActive:(UIApplication *)application { - // Restart any tasks that were paused (or not yet started) while the application was inactive. If - // the application was previously in the background, optionally refresh the user interface. -} - -- (void)applicationWillTerminate:(UIApplication *)application { - // Called when the application is about to terminate. Save data if appropriate. See also - // applicationDidEnterBackground:. -} - -@end diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_1x.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_1x.png deleted file mode 100644 index ed44f758e..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_1x.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_2x.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_2x.png deleted file mode 100644 index 5855920a6..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_2x.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_3x.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_3x.png deleted file mode 100644 index c96e2f0aa..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/40_c_3x.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_2x.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_2x.png deleted file mode 100644 index 3037b13eb..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_2x.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_3x.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_3x.png deleted file mode 100644 index 1e905ec27..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/60_c_iphone_3x.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/76_c_Ipad.png b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/76_c_Ipad.png deleted file mode 100644 index d28effc39..000000000 Binary files a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/76_c_Ipad.png and /dev/null differ diff --git a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/Contents.json b/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/Contents.json deleted file mode 100644 index 8ae934c76..000000000 --- a/mediapipe/examples/ios/common/Assets.xcassets/AppIcon.appiconset/Contents.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "images" : [ - { - "idiom" : "iphone", - "size" : "20x20", - "scale" : "2x" - }, - { - "idiom" : "iphone", - "size" : "20x20", - "scale" : "3x" - }, - { - "idiom" : "iphone", - "size" : "29x29", - "scale" : "2x" - }, - { - "idiom" : "iphone", - "size" : "29x29", - "scale" : "3x" - }, - { - "idiom" : "iphone", - "size" : "40x40", - "filename" : "40_c_2x.png", - "scale" : "2x" - }, - { - "idiom" : "iphone", - "size" : "40x40", - "filename" : "40_c_3x.png", - "scale" : "3x" - }, - { - "idiom" : "iphone", - "size" : "60x60", - "filename" : "60_c_iphone_2x.png", - "scale" : "2x" - }, - { - "idiom" : "iphone", - "size" : "60x60", - "filename" : "60_c_iphone_3x.png", - "scale" : "3x" - }, - { - "idiom" : "ipad", - "size" : "20x20", - "scale" : "1x" - }, - { - "idiom" : "ipad", - "size" : "20x20", - "scale" : "2x" - }, - { - "idiom" : "ipad", - "size" : "29x29", - "scale" : "1x" - }, - { - "idiom" : "ipad", - "size" : "29x29", - "scale" : "2x" - }, - { - "idiom" : "ipad", - "size" : "40x40", - "filename" : "40_c_1x.png", - "scale" : "1x" - }, - { - "idiom" : "ipad", - "size" : "40x40", - "scale" : "2x" - }, - { - "idiom" : "ipad", - "size" : "76x76", - "filename" : "76_c_Ipad.png", - "scale" : "1x" - }, - { - "idiom" : "ipad", - "size" : "76x76", - "scale" : "2x" - }, - { - "idiom" : "ipad", - "size" : "83.5x83.5", - "scale" : "2x" - }, - { - "idiom" : "ios-marketing", - "size" : "1024x1024", - "scale" : "1x" - } - ], - "info" : { - "version" : 1, - "author" : "xcode" - } -} - diff --git a/mediapipe/examples/ios/common/Assets.xcassets/Contents.json b/mediapipe/examples/ios/common/Assets.xcassets/Contents.json deleted file mode 100644 index 7afcdfaf8..000000000 --- a/mediapipe/examples/ios/common/Assets.xcassets/Contents.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "info" : { - "version" : 1, - "author" : "xcode" - } -} - diff --git a/mediapipe/examples/ios/common/BUILD b/mediapipe/examples/ios/common/BUILD deleted file mode 100644 index 0f3d34cd1..000000000 --- a/mediapipe/examples/ios/common/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) # Apache 2.0 - -objc_library( - name = "CommonMediaPipeAppLibrary", - srcs = [ - "AppDelegate.mm", - "CommonViewController.mm", - "main.m", - ], - hdrs = [ - "AppDelegate.h", - "CommonViewController.h", - ], - data = [ - "Base.lproj/LaunchScreen.storyboard", - "Base.lproj/Main.storyboard", - ], - sdk_frameworks = [ - "AVFoundation", - "CoreGraphics", - "CoreMedia", - "UIKit", - ], - visibility = ["//mediapipe:__subpackages__"], - deps = [ - "//mediapipe/objc:mediapipe_framework_ios", - "//mediapipe/objc:mediapipe_input_sources_ios", - "//mediapipe/objc:mediapipe_layer_renderer", - ], -) - -exports_files(["Info.plist"]) - -filegroup( - name = "AppIcon", - srcs = glob(["Assets.xcassets/AppIcon.appiconset/*"]), - visibility = ["//mediapipe:__subpackages__"], -) diff --git a/mediapipe/examples/ios/common/Base.lproj/LaunchScreen.storyboard b/mediapipe/examples/ios/common/Base.lproj/LaunchScreen.storyboard deleted file mode 100644 index bfa361294..000000000 --- a/mediapipe/examples/ios/common/Base.lproj/LaunchScreen.storyboard +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/ios/common/Base.lproj/Main.storyboard b/mediapipe/examples/ios/common/Base.lproj/Main.storyboard deleted file mode 100644 index fcf71c0e2..000000000 --- a/mediapipe/examples/ios/common/Base.lproj/Main.storyboard +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/ios/common/CommonViewController.h b/mediapipe/examples/ios/common/CommonViewController.h deleted file mode 100644 index d7cb1121a..000000000 --- a/mediapipe/examples/ios/common/CommonViewController.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/objc/MPPCameraInputSource.h" -#import "mediapipe/objc/MPPGraph.h" -#import "mediapipe/objc/MPPLayerRenderer.h" -#import "mediapipe/objc/MPPPlayerInputSource.h" -#import "mediapipe/objc/MPPTimestampConverter.h" - -typedef NS_ENUM(NSInteger, MediaPipeDemoSourceMode) { - MediaPipeDemoSourceCamera, - MediaPipeDemoSourceVideo -}; - -@interface CommonViewController : UIViewController - -// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in -// viewWillAppear: and sent video frames on videoQueue. -@property(nonatomic) MPPGraph* mediapipeGraph; - -// Handles camera access via AVCaptureSession library. -@property(nonatomic) MPPCameraInputSource* cameraSource; - -// Provides data from a video. -@property(nonatomic) MPPPlayerInputSource* videoSource; - -// Helps to convert timestamp. -@property(nonatomic) MPPTimestampConverter* timestampConverter; - -// The data source for the demo. -@property(nonatomic) MediaPipeDemoSourceMode sourceMode; - -// Inform the user when camera is unavailable. -@property(nonatomic) IBOutlet UILabel* noCameraLabel; - -// Display the camera preview frames. -@property(strong, nonatomic) IBOutlet UIView* liveView; - -// Render frames in a layer. -@property(nonatomic) MPPLayerRenderer* renderer; - -// Process camera frames on this queue. -@property(nonatomic) dispatch_queue_t videoQueue; - -// Graph name. -@property(nonatomic) NSString* graphName; - -// Graph input stream. -@property(nonatomic) const char* graphInputStream; - -// Graph output stream. -@property(nonatomic) const char* graphOutputStream; - -@end diff --git a/mediapipe/examples/ios/common/CommonViewController.mm b/mediapipe/examples/ios/common/CommonViewController.mm deleted file mode 100644 index f6c47eacf..000000000 --- a/mediapipe/examples/ios/common/CommonViewController.mm +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "CommonViewController.h" - -static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue"; - -@implementation CommonViewController - -// This provides a hook to replace the basic ViewController with a subclass when it's created from a -// storyboard, without having to change the storyboard itself. -+ (instancetype)allocWithZone:(struct _NSZone*)zone { - NSString* subclassName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"MainViewController"]; - if (subclassName.length > 0) { - Class customClass = NSClassFromString(subclassName); - Class baseClass = [CommonViewController class]; - NSAssert([customClass isSubclassOfClass:baseClass], @"%@ must be a subclass of %@", customClass, - baseClass); - if (self == baseClass) return [customClass allocWithZone:zone]; - } - return [super allocWithZone:zone]; -} - -#pragma mark - Cleanup methods - -- (void)dealloc { - self.mediapipeGraph.delegate = nil; - [self.mediapipeGraph cancel]; - // Ignore errors since we're cleaning up. - [self.mediapipeGraph closeAllInputStreamsWithError:nil]; - [self.mediapipeGraph waitUntilDoneWithError:nil]; -} - -#pragma mark - MediaPipe graph methods - -+ (MPPGraph*)loadGraphFromResource:(NSString*)resource { - // Load the graph config resource. - NSError* configLoadError = nil; - NSBundle* bundle = [NSBundle bundleForClass:[self class]]; - if (!resource || resource.length == 0) { - return nil; - } - NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; - NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; - if (!data) { - NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); - return nil; - } - - // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. - mediapipe::CalculatorGraphConfig config; - config.ParseFromArray(data.bytes, data.length); - - // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. - MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; - return newGraph; -} - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - self.renderer = [[MPPLayerRenderer alloc] init]; - self.renderer.layer.frame = self.liveView.layer.bounds; - [self.liveView.layer addSublayer:self.renderer.layer]; - self.renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop; - - self.timestampConverter = [[MPPTimestampConverter alloc] init]; - - dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class( - DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0); - self.videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute); - - self.graphName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphName"]; - self.graphInputStream = - [[[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphInputStream"] UTF8String]; - self.graphOutputStream = - [[[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphOutputStream"] UTF8String]; - - self.mediapipeGraph = [[self class] loadGraphFromResource:self.graphName]; - [self.mediapipeGraph addFrameOutputStream:self.graphOutputStream - outputPacketType:MPPPacketTypePixelBuffer]; - - self.mediapipeGraph.delegate = self; -} - -// In this application, there is only one ViewController which has no navigation to other view -// controllers, and there is only one View with live display showing the result of running the -// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph -// setup/teardown and camera start/stop logic should be updated appropriately in response to the -// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times -// depending on the application navigation flow in that case. -- (void)viewWillAppear:(BOOL)animated { - [super viewWillAppear:animated]; - - switch (self.sourceMode) { - case MediaPipeDemoSourceVideo: { - NSString* videoName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"VideoName"]; - AVAsset* video = [AVAsset assetWithURL:[[NSBundle mainBundle] URLForResource:videoName - withExtension:@"mov"]]; - self.videoSource = [[MPPPlayerInputSource alloc] initWithAVAsset:video]; - [self.videoSource setDelegate:self queue:self.videoQueue]; - dispatch_async(self.videoQueue, ^{ - [self.videoSource start]; - }); - break; - } - case MediaPipeDemoSourceCamera: { - self.cameraSource = [[MPPCameraInputSource alloc] init]; - [self.cameraSource setDelegate:self queue:self.videoQueue]; - self.cameraSource.sessionPreset = AVCaptureSessionPresetHigh; - - NSString* cameraPosition = - [[NSBundle mainBundle] objectForInfoDictionaryKey:@"CameraPosition"]; - if (cameraPosition.length > 0 && [cameraPosition isEqualToString:@"back"]) { - self.cameraSource.cameraPosition = AVCaptureDevicePositionBack; - } else { - self.cameraSource.cameraPosition = AVCaptureDevicePositionFront; - // When using the front camera, mirror the input for a more natural look. - _cameraSource.videoMirrored = YES; - } - - // The frame's native format is rotated with respect to the portrait orientation. - _cameraSource.orientation = AVCaptureVideoOrientationPortrait; - - [self.cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) { - if (granted) { - dispatch_async(dispatch_get_main_queue(), ^{ - self.noCameraLabel.hidden = YES; - }); - [self startGraphAndCamera]; - } - }]; - - break; - } - } -} - -- (void)startGraphAndCamera { - // Start running self.mediapipeGraph. - NSError* error; - if (![self.mediapipeGraph startWithError:&error]) { - NSLog(@"Failed to start graph: %@", error); - } - else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) { - NSLog(@"Failed to complete graph initial run: %@", error); - } - - // Start fetching frames from the camera. - dispatch_async(self.videoQueue, ^{ - [self.cameraSource start]; - }); -} - -#pragma mark - MPPInputSourceDelegate methods - -// Must be invoked on self.videoQueue. -- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer - timestamp:(CMTime)timestamp - fromSource:(MPPInputSource*)source { - if (source != self.cameraSource && source != self.videoSource) { - NSLog(@"Unknown source: %@", source); - return; - } - - [self.mediapipeGraph sendPixelBuffer:imageBuffer - intoStream:self.graphInputStream - packetType:MPPPacketTypePixelBuffer - timestamp:[self.timestampConverter timestampForMediaTime:timestamp]]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer - fromStream:(const std::string&)streamName { - if (streamName == self.graphOutputStream) { - // Display the captured image on the screen. - CVPixelBufferRetain(pixelBuffer); - dispatch_async(dispatch_get_main_queue(), ^{ - [self.renderer renderPixelBuffer:pixelBuffer]; - CVPixelBufferRelease(pixelBuffer); - }); - } -} - -@end diff --git a/mediapipe/examples/ios/common/Info.plist b/mediapipe/examples/ios/common/Info.plist deleted file mode 100644 index 30db14c62..000000000 --- a/mediapipe/examples/ios/common/Info.plist +++ /dev/null @@ -1,42 +0,0 @@ - - - - - NSCameraUsageDescription - This app uses the camera to demonstrate live video processing. - CFBundleDevelopmentRegion - en - CFBundleExecutable - $(EXECUTABLE_NAME) - CFBundleIdentifier - $(PRODUCT_BUNDLE_IDENTIFIER) - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - $(PRODUCT_NAME) - CFBundlePackageType - APPL - CFBundleShortVersionString - 1.0 - CFBundleVersion - 1 - LSRequiresIPhoneOS - - UILaunchStoryboardName - LaunchScreen - UIMainStoryboardFile - Main - UIRequiredDeviceCapabilities - - armv7 - - UISupportedInterfaceOrientations - - UIInterfaceOrientationPortrait - - UISupportedInterfaceOrientations~ipad - - UIInterfaceOrientationPortrait - - - diff --git a/mediapipe/examples/ios/common/main.m b/mediapipe/examples/ios/common/main.m deleted file mode 100644 index 7ffe5ea5d..000000000 --- a/mediapipe/examples/ios/common/main.m +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import -#import "AppDelegate.h" - -int main(int argc, char * argv[]) { - @autoreleasepool { - return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class])); - } -} diff --git a/mediapipe/examples/ios/facedetectioncpu/BUILD b/mediapipe/examples/ios/facedetectioncpu/BUILD deleted file mode 100644 index 43bff9b1e..000000000 --- a/mediapipe/examples/ios/facedetectioncpu/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "facedetectioncpu", - actual = "FaceDetectionCpuApp", -) - -ios_application( - name = "FaceDetectionCpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".FaceDetectionCpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":FaceDetectionCpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "FaceDetectionCpuAppLibrary", - data = [ - "//mediapipe/graphs/face_detection:face_detection_mobile_cpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/face_detection:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/facedetectioncpu/Info.plist b/mediapipe/examples/ios/facedetectioncpu/Info.plist deleted file mode 100644 index 34e1a7eee..000000000 --- a/mediapipe/examples/ios/facedetectioncpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - front - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - face_detection_mobile_cpu - - diff --git a/mediapipe/examples/ios/facedetectiongpu/BUILD b/mediapipe/examples/ios/facedetectiongpu/BUILD deleted file mode 100644 index 51856a7f7..000000000 --- a/mediapipe/examples/ios/facedetectiongpu/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "facedetectiongpu", - actual = "FaceDetectionGpuApp", -) - -ios_application( - name = "FaceDetectionGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".FaceDetectionGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":FaceDetectionGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "FaceDetectionGpuAppLibrary", - data = [ - "//mediapipe/graphs/face_detection:face_detection_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/face_detection:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/facedetectiongpu/Info.plist b/mediapipe/examples/ios/facedetectiongpu/Info.plist deleted file mode 100644 index 45feefb45..000000000 --- a/mediapipe/examples/ios/facedetectiongpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - front - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - face_detection_mobile_gpu - - diff --git a/mediapipe/examples/ios/faceeffect/AppDelegate.h b/mediapipe/examples/ios/faceeffect/AppDelegate.h deleted file mode 100644 index f7c48321c..000000000 --- a/mediapipe/examples/ios/faceeffect/AppDelegate.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -@interface AppDelegate : UIResponder - -@property(strong, nonatomic) UIWindow *window; - -@end diff --git a/mediapipe/examples/ios/faceeffect/AppDelegate.m b/mediapipe/examples/ios/faceeffect/AppDelegate.m deleted file mode 100644 index 42f9acd54..000000000 --- a/mediapipe/examples/ios/faceeffect/AppDelegate.m +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "AppDelegate.h" - -@interface AppDelegate () - -@end - -@implementation AppDelegate - -- (BOOL)application:(UIApplication *)application - didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { - // Override point for customization after application launch. - return YES; -} - -- (void)applicationWillResignActive:(UIApplication *)application { - // Sent when the application is about to move from active to inactive state. This can occur for - // certain types of temporary interruptions (such as an incoming phone call or SMS message) or - // when the user quits the application and it begins the transition to the background state. Use - // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering - // callbacks. Games should use this method to pause the game. -} - -- (void)applicationDidEnterBackground:(UIApplication *)application { - // Use this method to release shared resources, save user data, invalidate timers, and store - // enough application state information to restore your application to its current state in case - // it is terminated later. If your application supports background execution, this method is - // called instead of applicationWillTerminate: when the user quits. -} - -- (void)applicationWillEnterForeground:(UIApplication *)application { - // Called as part of the transition from the background to the active state; here you can undo - // many of the changes made on entering the background. -} - -- (void)applicationDidBecomeActive:(UIApplication *)application { - // Restart any tasks that were paused (or not yet started) while the application was inactive. If - // the application was previously in the background, optionally refresh the user interface. -} - -- (void)applicationWillTerminate:(UIApplication *)application { - // Called when the application is about to terminate. Save data if appropriate. See also - // applicationDidEnterBackground:. -} - -@end diff --git a/mediapipe/examples/ios/faceeffect/BUILD b/mediapipe/examples/ios/faceeffect/BUILD deleted file mode 100644 index 9e074ef2f..000000000 --- a/mediapipe/examples/ios/faceeffect/BUILD +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "faceeffect", - actual = "FaceEffectApp", -) - -ios_application( - name = "FaceEffectApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".FaceEffectGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = ["Info.plist"], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":FaceEffectAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "FaceEffectViewController", - srcs = [ - "FaceEffectViewController.mm", - ], - hdrs = [ - "FaceEffectViewController.h", - ], - copts = ["-std=c++17"], - data = [ - "Base.lproj/LaunchScreen.storyboard", - "Base.lproj/Main.storyboard", - "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb", - "//mediapipe/graphs/face_effect/data:axis.binarypb", - "//mediapipe/graphs/face_effect/data:axis.pngblob", - "//mediapipe/graphs/face_effect/data:facepaint.pngblob", - "//mediapipe/graphs/face_effect/data:glasses.binarypb", - "//mediapipe/graphs/face_effect/data:glasses.pngblob", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - ], - sdk_frameworks = [ - "AVFoundation", - "CoreGraphics", - "CoreMedia", - "UIKit", - ], - deps = [ - "//mediapipe/objc:mediapipe_framework_ios", - "//mediapipe/objc:mediapipe_input_sources_ios", - "//mediapipe/objc:mediapipe_layer_renderer", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/framework/formats:matrix_data_cc_proto", - "//mediapipe/graphs/face_effect:face_effect_gpu_deps", - "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto", - ], - }), -) - -objc_library( - name = "FaceEffectAppLibrary", - srcs = [ - "AppDelegate.m", - "main.m", - ], - hdrs = [ - "AppDelegate.h", - ], - data = [ - "Base.lproj/LaunchScreen.storyboard", - "Base.lproj/Main.storyboard", - "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb", - "//mediapipe/graphs/face_effect/data:facepaint.pngblob", - "//mediapipe/graphs/face_effect/data:glasses.binarypb", - "//mediapipe/graphs/face_effect/data:glasses.pngblob", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - ], - deps = [ - ":FaceEffectViewController", - ], -) diff --git a/mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard b/mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard deleted file mode 100644 index bfa361294..000000000 --- a/mediapipe/examples/ios/faceeffect/Base.lproj/LaunchScreen.storyboard +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard b/mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard deleted file mode 100644 index d7a79730e..000000000 --- a/mediapipe/examples/ios/faceeffect/Base.lproj/Main.storyboard +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/examples/ios/faceeffect/FaceEffectViewController.h b/mediapipe/examples/ios/faceeffect/FaceEffectViewController.h deleted file mode 100644 index 5d863cbf2..000000000 --- a/mediapipe/examples/ios/faceeffect/FaceEffectViewController.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -@interface FaceEffectViewController : UIViewController - -@end diff --git a/mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm b/mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm deleted file mode 100644 index 56a895c69..000000000 --- a/mediapipe/examples/ios/faceeffect/FaceEffectViewController.mm +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "FaceEffectViewController.h" - -#import "mediapipe/objc/MPPCameraInputSource.h" -#import "mediapipe/objc/MPPGraph.h" -#import "mediapipe/objc/MPPLayerRenderer.h" - -#include -#include -#include - -#include "mediapipe/framework/formats/matrix_data.pb.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h" - -static NSString* const kGraphName = @"face_effect_gpu"; - -static const char* kInputStream = "input_video"; -static const char* kOutputStream = "output_video"; -static const char* kMultiFaceGeometryStream = "multi_face_geometry"; -static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue"; -static const char* kSelectedEffectIdInputStream = "selected_effect_id"; -static const char* kUseFaceDetectionInputSourceInputSidePacket = "use_face_detection_input_source"; - -static const BOOL kUseFaceDetectionInputSource = NO; -static const int kMatrixTranslationZIndex = 14; - -static const int kSelectedEffectIdAxis = 0; -static const int kSelectedEffectIdFacepaint = 1; -static const int kSelectedEffectIdGlasses = 2; - -@interface FaceEffectViewController () - -// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and -// sent video frames on _videoQueue. -@property(nonatomic) MPPGraph* graph; - -@end - -@implementation FaceEffectViewController { - /// Handle tap gestures. - UITapGestureRecognizer* _tapGestureRecognizer; - int _selectedEffectId; - - /// Handles camera access via AVCaptureSession library. - MPPCameraInputSource* _cameraSource; - - /// Inform the user when camera is unavailable. - IBOutlet UILabel* _noCameraLabel; - /// Inform the user about how to switch between effects. - UILabel* _effectSwitchingHintLabel; - /// Display the camera preview frames. - IBOutlet UIView* _liveView; - /// Render frames in a layer. - MPPLayerRenderer* _renderer; - - /// Process camera frames on this queue. - dispatch_queue_t _videoQueue; -} - -#pragma mark - Cleanup methods - -- (void)dealloc { - self.graph.delegate = nil; - [self.graph cancel]; - // Ignore errors since we're cleaning up. - [self.graph closeAllInputStreamsWithError:nil]; - [self.graph waitUntilDoneWithError:nil]; -} - -#pragma mark - MediaPipe graph methods - -+ (MPPGraph*)loadGraphFromResource:(NSString*)resource { - // Load the graph config resource. - NSError* configLoadError = nil; - NSBundle* bundle = [NSBundle bundleForClass:[self class]]; - if (!resource || resource.length == 0) { - return nil; - } - NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; - NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; - if (!data) { - NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); - return nil; - } - - // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. - mediapipe::CalculatorGraphConfig config; - config.ParseFromArray(data.bytes, data.length); - - // Pass the kUseFaceDetectionInputSource flag value as an input side packet into the graph. - std::map side_packets; - side_packets[kUseFaceDetectionInputSourceInputSidePacket] = - mediapipe::MakePacket(kUseFaceDetectionInputSource); - - // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. - MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; - [newGraph addSidePackets:side_packets]; - [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer]; - [newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw]; - return newGraph; -} - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - _effectSwitchingHintLabel.hidden = YES; - _tapGestureRecognizer = [[UITapGestureRecognizer alloc] initWithTarget:self - action:@selector(handleTap)]; - [self.view addGestureRecognizer:_tapGestureRecognizer]; - - // By default, render the axis effect for the face detection input source and the glasses effect - // for the face landmark input source. - if (kUseFaceDetectionInputSource) { - _selectedEffectId = kSelectedEffectIdAxis; - } else { - _selectedEffectId = kSelectedEffectIdGlasses; - } - - _renderer = [[MPPLayerRenderer alloc] init]; - _renderer.layer.frame = _liveView.layer.bounds; - [_liveView.layer insertSublayer:_renderer.layer atIndex:0]; - _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop; - _renderer.mirrored = NO; - - dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class( - DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0); - _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute); - - _cameraSource = [[MPPCameraInputSource alloc] init]; - [_cameraSource setDelegate:self queue:_videoQueue]; - _cameraSource.sessionPreset = AVCaptureSessionPresetHigh; - _cameraSource.cameraPosition = AVCaptureDevicePositionFront; - // The frame's native format is rotated with respect to the portrait orientation. - _cameraSource.orientation = AVCaptureVideoOrientationPortrait; - _cameraSource.videoMirrored = YES; - - self.graph = [[self class] loadGraphFromResource:kGraphName]; - self.graph.delegate = self; - // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing. - self.graph.maxFramesInFlight = 2; -} - -// In this application, there is only one ViewController which has no navigation to other view -// controllers, and there is only one View with live display showing the result of running the -// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph -// setup/teardown and camera start/stop logic should be updated appropriately in response to the -// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times -// depending on the application navigation flow in that case. -- (void)viewWillAppear:(BOOL)animated { - [super viewWillAppear:animated]; - - [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) { - if (granted) { - [self startGraphAndCamera]; - dispatch_async(dispatch_get_main_queue(), ^{ - _noCameraLabel.hidden = YES; - }); - } - }]; -} - -- (void)startGraphAndCamera { - // Start running self.graph. - NSError* error; - if (![self.graph startWithError:&error]) { - NSLog(@"Failed to start graph: %@", error); - } - - // Start fetching frames from the camera. - dispatch_async(_videoQueue, ^{ - [_cameraSource start]; - }); -} - -#pragma mark - UITapGestureRecognizer methods - -// We use the tap gesture recognizer to switch between face effects. This allows users to try -// multiple pre-bundled face effects without a need to recompile the app. -- (void)handleTap { - dispatch_async(_videoQueue, ^{ - // Avoid switching the Axis effect for the face detection input source. - if (kUseFaceDetectionInputSource) { - return; - } - - // Looped effect order: glasses -> facepaint -> axis -> glasses -> ... - switch (_selectedEffectId) { - case kSelectedEffectIdAxis: { - _selectedEffectId = kSelectedEffectIdGlasses; - break; - } - - case kSelectedEffectIdFacepaint: { - _selectedEffectId = kSelectedEffectIdAxis; - break; - } - - case kSelectedEffectIdGlasses: { - _selectedEffectId = kSelectedEffectIdFacepaint; - break; - } - } - }); -} - -#pragma mark - MPPGraphDelegate methods - -// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer - fromStream:(const std::string&)streamName { - if (streamName == kOutputStream) { - // Display the captured image on the screen. - CVPixelBufferRetain(pixelBuffer); - dispatch_async(dispatch_get_main_queue(), ^{ - _effectSwitchingHintLabel.hidden = kUseFaceDetectionInputSource; - [_renderer renderPixelBuffer:pixelBuffer]; - CVPixelBufferRelease(pixelBuffer); - }); - } -} - -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -// -// This callback demonstrates how the output face geometry packet can be obtained and used in an -// iOS app. As an example, the Z-translation component of the face pose transform matrix is logged -// for each face being equal to the approximate distance away from the camera in centimeters. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kMultiFaceGeometryStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No face geometry", packet.Timestamp().Value()); - return; - } - - const auto& multiFaceGeometry = - packet.Get>(); - NSLog(@"[TS:%lld] Number of face instances with geometry: %lu ", packet.Timestamp().Value(), - multiFaceGeometry.size()); - for (int faceIndex = 0; faceIndex < multiFaceGeometry.size(); ++faceIndex) { - const auto& faceGeometry = multiFaceGeometry[faceIndex]; - NSLog(@"\tApprox. distance away from camera for face[%d]: %.6f cm", faceIndex, - -faceGeometry.pose_transform_matrix().packed_data(kMatrixTranslationZIndex)); - } - } -} - -#pragma mark - MPPInputSourceDelegate methods - -// Must be invoked on _videoQueue. -- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer - timestamp:(CMTime)timestamp - fromSource:(MPPInputSource*)source { - if (source != _cameraSource) { - NSLog(@"Unknown source: %@", source); - return; - } - - mediapipe::Timestamp graphTimestamp(static_cast( - mediapipe::Timestamp::kTimestampUnitsPerSecond * CMTimeGetSeconds(timestamp))); - - mediapipe::Packet selectedEffectIdPacket = - mediapipe::MakePacket(_selectedEffectId).At(graphTimestamp); - - [self.graph sendPixelBuffer:imageBuffer - intoStream:kInputStream - packetType:MPPPacketTypePixelBuffer - timestamp:graphTimestamp]; - - // Alongside the input camera frame, we also send the `selected_effect_id` int packet to indicate - // which effect should be rendered on this frame. - [self.graph movePacket:std::move(selectedEffectIdPacket) - intoStream:kSelectedEffectIdInputStream - error:nil]; -} - -@end diff --git a/mediapipe/examples/ios/faceeffect/Info.plist b/mediapipe/examples/ios/faceeffect/Info.plist deleted file mode 100644 index 30db14c62..000000000 --- a/mediapipe/examples/ios/faceeffect/Info.plist +++ /dev/null @@ -1,42 +0,0 @@ - - - - - NSCameraUsageDescription - This app uses the camera to demonstrate live video processing. - CFBundleDevelopmentRegion - en - CFBundleExecutable - $(EXECUTABLE_NAME) - CFBundleIdentifier - $(PRODUCT_BUNDLE_IDENTIFIER) - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - $(PRODUCT_NAME) - CFBundlePackageType - APPL - CFBundleShortVersionString - 1.0 - CFBundleVersion - 1 - LSRequiresIPhoneOS - - UILaunchStoryboardName - LaunchScreen - UIMainStoryboardFile - Main - UIRequiredDeviceCapabilities - - armv7 - - UISupportedInterfaceOrientations - - UIInterfaceOrientationPortrait - - UISupportedInterfaceOrientations~ipad - - UIInterfaceOrientationPortrait - - - diff --git a/mediapipe/examples/ios/faceeffect/main.m b/mediapipe/examples/ios/faceeffect/main.m deleted file mode 100644 index 8848aeef4..000000000 --- a/mediapipe/examples/ios/faceeffect/main.m +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import -#import "AppDelegate.h" - -int main(int argc, char* argv[]) { - @autoreleasepool { - return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class])); - } -} diff --git a/mediapipe/examples/ios/facemeshgpu/BUILD b/mediapipe/examples/ios/facemeshgpu/BUILD deleted file mode 100644 index 942a19659..000000000 --- a/mediapipe/examples/ios/facemeshgpu/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "facemeshgpu", - actual = "FaceMeshGpuApp", -) - -ios_application( - name = "FaceMeshGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":FaceMeshGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "FaceMeshGpuAppLibrary", - srcs = [ - "FaceMeshGpuViewController.mm", - ], - hdrs = [ - "FaceMeshGpuViewController.h", - ], - copts = ["-std=c++17"], - data = [ - "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/face_mesh:mobile_calculators", - "//mediapipe/framework/formats:landmark_cc_proto", - ], - }), -) diff --git a/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.h b/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.h deleted file mode 100644 index 520940f59..000000000 --- a/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/examples/ios/common/CommonViewController.h" - -@interface FaceMeshGpuViewController : CommonViewController - -@end diff --git a/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm b/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm deleted file mode 100644 index 6f11be42b..000000000 --- a/mediapipe/examples/ios/facemeshgpu/FaceMeshGpuViewController.mm +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "FaceMeshGpuViewController.h" - -#include "mediapipe/framework/formats/landmark.pb.h" - -static NSString* const kGraphName = @"face_mesh_mobile_gpu"; - -static const char* kNumFacesInputSidePacket = "num_faces"; -static const char* kLandmarksOutputStream = "multi_face_landmarks"; - -// Max number of faces to detect/process. -static const int kNumFaces = 1; - -@implementation FaceMeshGpuViewController - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - [self.mediapipeGraph setSidePacket:(mediapipe::MakePacket(kNumFaces)) - named:kNumFacesInputSidePacket]; - [self.mediapipeGraph addFrameOutputStream:kLandmarksOutputStream - outputPacketType:MPPPacketTypeRaw]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kLandmarksOutputStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value()); - return; - } - const auto& multi_face_landmarks = packet.Get>(); - NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(), - multi_face_landmarks.size()); - for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) { - const auto& landmarks = multi_face_landmarks[face_index]; - NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size()); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), - landmarks.landmark(i).y(), landmarks.landmark(i).z()); - } - } - } -} - -@end diff --git a/mediapipe/examples/ios/facemeshgpu/Info.plist b/mediapipe/examples/ios/facemeshgpu/Info.plist deleted file mode 100644 index 2684c8cab..000000000 --- a/mediapipe/examples/ios/facemeshgpu/Info.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - CameraPosition - front - MainViewController - FaceMeshGpuViewController - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - face_mesh_mobile_gpu - - diff --git a/mediapipe/examples/ios/handdetectiongpu/BUILD b/mediapipe/examples/ios/handdetectiongpu/BUILD deleted file mode 100644 index 1b5ed9820..000000000 --- a/mediapipe/examples/ios/handdetectiongpu/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "handdetectiongpu", - actual = "HandDetectionGpuApp", -) - -ios_application( - name = "HandDetectionGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".HandDetectionGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":HandDetectionGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "HandDetectionGpuAppLibrary", - data = [ - "//mediapipe/graphs/hand_tracking:hand_detection_mobile_gpu_binary_graph", - "//mediapipe/modules/palm_detection:palm_detection.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/hand_tracking:detection_mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/handdetectiongpu/Info.plist b/mediapipe/examples/ios/handdetectiongpu/Info.plist deleted file mode 100644 index 937a29569..000000000 --- a/mediapipe/examples/ios/handdetectiongpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - front - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - hand_detection_mobile_gpu - - diff --git a/mediapipe/examples/ios/handtrackinggpu/BUILD b/mediapipe/examples/ios/handtrackinggpu/BUILD deleted file mode 100644 index 0121150e1..000000000 --- a/mediapipe/examples/ios/handtrackinggpu/BUILD +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "handtrackinggpu", - actual = "HandTrackingGpuApp", -) - -ios_application( - name = "HandTrackingGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".HandTrackingGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":HandTrackingGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "HandTrackingGpuAppLibrary", - srcs = [ - "HandTrackingViewController.mm", - ], - hdrs = [ - "HandTrackingViewController.h", - ], - copts = ["-std=c++17"], - data = [ - "//mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu.binarypb", - "//mediapipe/modules/hand_landmark:hand_landmark.tflite", - "//mediapipe/modules/hand_landmark:handedness.txt", - "//mediapipe/modules/palm_detection:palm_detection.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/hand_tracking:mobile_calculators", - "//mediapipe/framework/formats:landmark_cc_proto", - ], - }), -) diff --git a/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.h b/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.h deleted file mode 100644 index a3d521f22..000000000 --- a/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/examples/ios/common/CommonViewController.h" - -@interface HandTrackingViewController : CommonViewController - -@end diff --git a/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.mm b/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.mm deleted file mode 100644 index 87e562d01..000000000 --- a/mediapipe/examples/ios/handtrackinggpu/HandTrackingViewController.mm +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "HandTrackingViewController.h" - -#include "mediapipe/framework/formats/landmark.pb.h" - -static const char* kLandmarksOutputStream = "hand_landmarks"; -static const char* kNumHandsInputSidePacket = "num_hands"; - -// Max number of hands to detect/process. -static const int kNumHands = 2; - -@implementation HandTrackingViewController - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - [self.mediapipeGraph setSidePacket:(mediapipe::MakePacket(kNumHands)) - named:kNumHandsInputSidePacket]; - [self.mediapipeGraph addFrameOutputStream:kLandmarksOutputStream - outputPacketType:MPPPacketTypeRaw]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kLandmarksOutputStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No hand landmarks", packet.Timestamp().Value()); - return; - } - const auto& multiHandLandmarks = packet.Get>(); - NSLog(@"[TS:%lld] Number of hand instances with landmarks: %lu", packet.Timestamp().Value(), - multiHandLandmarks.size()); - for (int handIndex = 0; handIndex < multiHandLandmarks.size(); ++handIndex) { - const auto& landmarks = multiHandLandmarks[handIndex]; - NSLog(@"\tNumber of landmarks for hand[%d]: %d", handIndex, landmarks.landmark_size()); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), - landmarks.landmark(i).y(), landmarks.landmark(i).z()); - } - } - } -} - -@end diff --git a/mediapipe/examples/ios/handtrackinggpu/Info.plist b/mediapipe/examples/ios/handtrackinggpu/Info.plist deleted file mode 100644 index a5eebbefa..000000000 --- a/mediapipe/examples/ios/handtrackinggpu/Info.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - CameraPosition - front - MainViewController - HandTrackingViewController - GraphInputStream - input_video - GraphOutputStream - output_video - GraphName - hand_tracking_mobile_gpu - - diff --git a/mediapipe/examples/ios/helloworld/BUILD b/mediapipe/examples/ios/helloworld/BUILD deleted file mode 100644 index 192996bf3..000000000 --- a/mediapipe/examples/ios/helloworld/BUILD +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "helloworld", - actual = "HelloWorldApp", -) - -ios_application( - name = "HelloWorldApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".HelloWorld", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [":HelloWorldAppLibrary"], -) - -objc_library( - name = "HelloWorldAppLibrary", - data = [ - "//mediapipe/graphs/edge_detection:mobile_gpu_binary_graph", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - "//mediapipe/graphs/edge_detection:mobile_calculators", - ], -) diff --git a/mediapipe/examples/ios/helloworld/Info.plist b/mediapipe/examples/ios/helloworld/Info.plist deleted file mode 100644 index 7e792c9b4..000000000 --- a/mediapipe/examples/ios/helloworld/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - back - GraphName - mobile_gpu - GraphOutputStream - output_video - GraphInputStream - input_video - - diff --git a/mediapipe/examples/ios/holistictrackinggpu/BUILD b/mediapipe/examples/ios/holistictrackinggpu/BUILD deleted file mode 100644 index b080564ce..000000000 --- a/mediapipe/examples/ios/holistictrackinggpu/BUILD +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "holistictrackinggpu", - actual = "HolisticTrackingGpuApp", -) - -ios_application( - name = "HolisticTrackingGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".HolisticTrackingGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":HolisticTrackingGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "HolisticTrackingGpuAppLibrary", - data = [ - "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/hand_landmark:hand_landmark.tflite", - "//mediapipe/modules/hand_landmark:handedness.txt", - "//mediapipe/modules/holistic_landmark:hand_recrop.tflite", - "//mediapipe/modules/pose_detection:pose_detection.tflite", - "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/holistic_tracking:holistic_tracking_gpu_deps", - ], - }), -) diff --git a/mediapipe/examples/ios/holistictrackinggpu/Info.plist b/mediapipe/examples/ios/holistictrackinggpu/Info.plist deleted file mode 100644 index ae92eb50f..000000000 --- a/mediapipe/examples/ios/holistictrackinggpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - back - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - holistic_tracking_gpu - - diff --git a/mediapipe/examples/ios/iristrackinggpu/BUILD b/mediapipe/examples/ios/iristrackinggpu/BUILD deleted file mode 100644 index b58ecc104..000000000 --- a/mediapipe/examples/ios/iristrackinggpu/BUILD +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "iristrackinggpu", - actual = "IrisTrackingGpuApp", -) - -ios_application( - name = "IrisTrackingGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".IrisTrackingGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":IrisTrackingGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "IrisTrackingGpuAppLibrary", - srcs = [ - "IrisTrackingViewController.mm", - ], - hdrs = [ - "IrisTrackingViewController.h", - ], - copts = ["-std=c++17"], - data = [ - "//mediapipe/graphs/iris_tracking:iris_tracking_gpu.binarypb", - "//mediapipe/modules/face_detection:face_detection_front.tflite", - "//mediapipe/modules/face_landmark:face_landmark.tflite", - "//mediapipe/modules/iris_landmark:iris_landmark.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/iris_tracking:iris_tracking_gpu_deps", - "//mediapipe/framework/formats:landmark_cc_proto", - ], - }), -) diff --git a/mediapipe/examples/ios/iristrackinggpu/Info.plist b/mediapipe/examples/ios/iristrackinggpu/Info.plist deleted file mode 100644 index aadef04ae..000000000 --- a/mediapipe/examples/ios/iristrackinggpu/Info.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - CameraPosition - front - MainViewController - IrisTrackingViewController - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - iris_tracking_gpu - - diff --git a/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.h b/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.h deleted file mode 100644 index ce133f330..000000000 --- a/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/examples/ios/common/CommonViewController.h" - -@interface IrisTrackingViewController : CommonViewController - -@end diff --git a/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.mm b/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.mm deleted file mode 100644 index 52687efa9..000000000 --- a/mediapipe/examples/ios/iristrackinggpu/IrisTrackingViewController.mm +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "IrisTrackingViewController.h" - -#include "mediapipe/framework/formats/landmark.pb.h" - -static const char* kLandmarksOutputStream = "iris_landmarks"; - -@implementation IrisTrackingViewController { - /// Input side packet for focal length parameter. - std::map _input_side_packets; - mediapipe::Packet _focal_length_side_packet; -} - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - [self.mediapipeGraph addFrameOutputStream:kLandmarksOutputStream - outputPacketType:MPPPacketTypeRaw]; - _focal_length_side_packet = - mediapipe::MakePacket>(absl::make_unique(0.0)); - _input_side_packets = { - {"focal_length_pixel", _focal_length_side_packet}, - }; - [self.mediapipeGraph addSidePackets:_input_side_packets]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kLandmarksOutputStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No iris landmarks", packet.Timestamp().Value()); - return; - } - const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>(); - NSLog(@"[TS:%lld] Number of landmarks on iris: %d", packet.Timestamp().Value(), - landmarks.landmark_size()); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - NSLog(@"\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), - landmarks.landmark(i).y(), landmarks.landmark(i).z()); - } - } -} - -#pragma mark - MPPInputSourceDelegate methods - -// Must be invoked on _videoQueue. -- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer - timestamp:(CMTime)timestamp - fromSource:(MPPInputSource*)source { - if (source != self.cameraSource) { - NSLog(@"Unknown source: %@", source); - return; - } - - // TODO: This is a temporary solution. Need to verify whether the focal length is - // constant. In that case, we need to use input stream instead of using side packet. - *(_input_side_packets["focal_length_pixel"].Get>()) = - self.cameraSource.cameraIntrinsicMatrix.columns[0][0]; - [self.mediapipeGraph sendPixelBuffer:imageBuffer - intoStream:self.graphInputStream - packetType:MPPPacketTypePixelBuffer]; -} - -@end diff --git a/mediapipe/examples/ios/link_local_profiles.py b/mediapipe/examples/ios/link_local_profiles.py deleted file mode 100755 index 9814009e1..000000000 --- a/mediapipe/examples/ios/link_local_profiles.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Lint as: python3 -"""This script is used to set up automatic provisioning for iOS examples. - -It scans the provisioning profiles used by Xcode, looking for one matching the -application identifier for each example app. If found, it symlinks the profile -in the appropriate location for Bazel to find it. - -It also checks whether the bundle_id.bzl file has a placeholder bundle ID, and -replaces it with a unique ID if so. -""" - -import os -import plistlib -import re -import subprocess -import uuid - -# This script is meant to be located in the MediaPipe iOS examples directory -# root. The logic below will have to be changed if the directory structure is -# reorganized. -examples_ios = os.path.dirname(os.path.realpath(__file__)) -example_names = { - f for f in os.listdir(examples_ios) - if os.path.isdir(os.path.join(examples_ios, f)) -} - - -def configure_bundle_id_prefix( - bundle_id_bzl=os.path.join(examples_ios, "bundle_id.bzl")) -> str: - """Configures the bundle id prefix to use. - - Gets the bundle id prefix in use from bundle_id.bzl; sets up a unique - prefix if not already set. - - Args: - bundle_id_bzl: Path to the bzl file where the bundle id is stored. - - Returns: - The bundle id prefix to use. - - Raises: - Exception: If parsing of bundle_id.bzl fails. - """ - bundle_id_re = re.compile( - r'^BUNDLE_ID_PREFIX\s*=\s*"(.*)"', flags=re.MULTILINE) - - with open(bundle_id_bzl, "r") as f: - contents = f.read() - match = bundle_id_re.search(contents) - if not match: - raise Exception("could not find BUNDLE_ID_PREFIX") - - bundle_id_prefix = match.group(1) - # The default value contains a *, which is an invalid character in bundle IDs. - if "*" in bundle_id_prefix: - bundle_id_prefix = str(uuid.uuid4()) + ".mediapipe.examples" - contents = contents[:match.start(1)] + bundle_id_prefix + contents[match - .end(1):] - with open(bundle_id_bzl, "w") as f: - f.write(contents) - print("Set up a unique bundle ID prefix: " + bundle_id_prefix) - - return bundle_id_prefix - - -def get_app_id(profile_path) -> str: - try: - plist = subprocess.check_output( - ["security", "cms", "-D", "-i", profile_path]) - profile = plistlib.loads(plist) - return profile["Entitlements"]["application-identifier"] - except Exception: # pylint: disable=broad-except - return None - - -def update_symlink(target_path, link_path): - if os.path.islink(link_path): - print(f" Removing existing symlink at {link_path}") - os.remove(link_path) - elif os.path.exists(link_path): - print(f" Unexpected existing file at {link_path}; skipping") - return - os.symlink(target_path, link_path) - print(f" Created symlink to {target_path} at {link_path}") - - -def process_profile(profile_path, our_app_id_re): - """Processes one mobileprovision file. - - Checks if its app ID matches one of our example apps, and symlinks it in the - appropriate location if so. - - Args: - profile_path: Path to the mobileprovision file. - our_app_id_re: Regular expression to extract the example name from one of - out app ids. - """ - app_id = get_app_id(profile_path) - if not app_id: - print(f"Could not parse '{profile_path}', skipping") - return - match = our_app_id_re.match(app_id) - if not match: - return - app_name = match.group(1) - app_dir_name = app_name.lower() - if app_dir_name not in example_names: - print(f"The app id '{app_id}' has our prefix, but does not seem to match" + - "any of our examples. Skipping.") - return - - print(f"Found profile for {app_name}") - - link_path = os.path.join(examples_ios, app_dir_name, - "provisioning_profile.mobileprovision") - update_symlink(profile_path, link_path) - - -def main(): - bundle_id_prefix = configure_bundle_id_prefix() - our_app_id_re = re.compile(r"[0-9A-Z]+\." + re.escape(bundle_id_prefix) + - r"\.(.*)") - - profile_dir = os.path.expanduser( - "~/Library/MobileDevice/Provisioning Profiles") - if not os.path.isdir(profile_dir): - print(f"Could not find provisioning profiles directory at {profile_dir}") - return 2 - - print( - f"Looking for profiles for app ids with prefix '{bundle_id_prefix}' in '{profile_dir}'" - ) - - for name in os.listdir(profile_dir): - if not name.endswith(".mobileprovision"): - continue - profile_path = os.path.join(profile_dir, name) - process_profile(profile_path, our_app_id_re) - - -if __name__ == "__main__": - main() diff --git a/mediapipe/examples/ios/objectdetectioncpu/BUILD b/mediapipe/examples/ios/objectdetectioncpu/BUILD deleted file mode 100644 index 5ddd12df6..000000000 --- a/mediapipe/examples/ios/objectdetectioncpu/BUILD +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "objectdetectioncpu", - actual = "ObjectDetectionCpuApp", -) - -ios_application( - name = "ObjectDetectionCpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".ObjectDetectionCpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":ObjectDetectionCpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "ObjectDetectionCpuAppLibrary", - data = [ - "//mediapipe/graphs/object_detection:mobile_cpu_binary_graph", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/object_detection:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/objectdetectioncpu/Info.plist b/mediapipe/examples/ios/objectdetectioncpu/Info.plist deleted file mode 100644 index e420121c8..000000000 --- a/mediapipe/examples/ios/objectdetectioncpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - back - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - mobile_cpu - - diff --git a/mediapipe/examples/ios/objectdetectiongpu/BUILD b/mediapipe/examples/ios/objectdetectiongpu/BUILD deleted file mode 100644 index b31c13f53..000000000 --- a/mediapipe/examples/ios/objectdetectiongpu/BUILD +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) # Apache 2.0 - -MIN_IOS_VERSION = "10.0" - -alias( - name = "objectdetectiongpu", - actual = "ObjectDetectionGpuApp", -) - -ios_application( - name = "ObjectDetectionGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".ObjectDetectionGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":ObjectDetectionGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "ObjectDetectionGpuAppLibrary", - data = [ - "//mediapipe/graphs/object_detection:mobile_gpu_binary_graph", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/object_detection:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/objectdetectiongpu/Info.plist b/mediapipe/examples/ios/objectdetectiongpu/Info.plist deleted file mode 100644 index 53930fe4a..000000000 --- a/mediapipe/examples/ios/objectdetectiongpu/Info.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - CameraPosition - back - GraphName - mobile_gpu - GraphOutputStream - output_video - GraphInputStream - input_video - VideoName - object_detection - - diff --git a/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD b/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD deleted file mode 100644 index 37e0b85e9..000000000 --- a/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "objectdetectiontrackinggpu", - actual = "ObjectDetectionTrackingGpuApp", -) - -ios_application( - name = "ObjectDetectionTrackingGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".ObjectDetectionTrackingGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":ObjectDetectionTrackingGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "ObjectDetectionTrackingGpuAppLibrary", - data = [ - "//mediapipe/graphs/tracking:mobile_gpu_binary_graph", - "//mediapipe/models:ssdlite_object_detection.tflite", - "//mediapipe/models:ssdlite_object_detection_labelmap.txt", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/tracking:mobile_calculators", - ], - }), -) diff --git a/mediapipe/examples/ios/objectdetectiontrackinggpu/Info.plist b/mediapipe/examples/ios/objectdetectiontrackinggpu/Info.plist deleted file mode 100644 index 7e792c9b4..000000000 --- a/mediapipe/examples/ios/objectdetectiontrackinggpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - back - GraphName - mobile_gpu - GraphOutputStream - output_video - GraphInputStream - input_video - - diff --git a/mediapipe/examples/ios/posetrackinggpu/BUILD b/mediapipe/examples/ios/posetrackinggpu/BUILD deleted file mode 100644 index 01a82cb4b..000000000 --- a/mediapipe/examples/ios/posetrackinggpu/BUILD +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2020 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "posetrackinggpu", - actual = "PoseTrackingGpuApp", -) - -ios_application( - name = "PoseTrackingGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".PoseTrackingGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":PoseTrackingGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "PoseTrackingGpuAppLibrary", - srcs = [ - "PoseTrackingViewController.mm", - ], - hdrs = [ - "PoseTrackingViewController.h", - ], - copts = ["-std=c++17"], - data = [ - "//mediapipe/graphs/pose_tracking:pose_tracking_gpu.binarypb", - "//mediapipe/modules/pose_detection:pose_detection.tflite", - "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/pose_tracking:pose_tracking_gpu_deps", - "//mediapipe/framework/formats:landmark_cc_proto", - ], - }), -) diff --git a/mediapipe/examples/ios/posetrackinggpu/Info.plist b/mediapipe/examples/ios/posetrackinggpu/Info.plist deleted file mode 100644 index 71e2e429e..000000000 --- a/mediapipe/examples/ios/posetrackinggpu/Info.plist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - CameraPosition - back - MainViewController - PoseTrackingViewController - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - pose_tracking_gpu - - diff --git a/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.h b/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.h deleted file mode 100644 index f5dc4674a..000000000 --- a/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/examples/ios/common/CommonViewController.h" - -@interface PoseTrackingViewController : CommonViewController - -@end diff --git a/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.mm b/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.mm deleted file mode 100644 index 0f082031c..000000000 --- a/mediapipe/examples/ios/posetrackinggpu/PoseTrackingViewController.mm +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2020 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "PoseTrackingViewController.h" - -#include "mediapipe/framework/formats/landmark.pb.h" - -static const char* kLandmarksOutputStream = "pose_landmarks"; - -@implementation PoseTrackingViewController - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - [self.mediapipeGraph addFrameOutputStream:kLandmarksOutputStream - outputPacketType:MPPPacketTypeRaw]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kLandmarksOutputStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No pose landmarks", packet.Timestamp().Value()); - return; - } - const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>(); - NSLog(@"[TS:%lld] Number of pose landmarks: %d", packet.Timestamp().Value(), - landmarks.landmark_size()); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - NSLog(@"\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), - landmarks.landmark(i).y(), landmarks.landmark(i).z()); - } - } -} - -@end diff --git a/mediapipe/examples/ios/selfiesegmentationgpu/BUILD b/mediapipe/examples/ios/selfiesegmentationgpu/BUILD deleted file mode 100644 index 884ac95a5..000000000 --- a/mediapipe/examples/ios/selfiesegmentationgpu/BUILD +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2021 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load( - "@build_bazel_rules_apple//apple:ios.bzl", - "ios_application", -) -load( - "//mediapipe/examples/ios:bundle_id.bzl", - "BUNDLE_ID_PREFIX", - "example_provisioning", -) - -licenses(["notice"]) - -MIN_IOS_VERSION = "10.0" - -alias( - name = "selfiesegmentationgpu", - actual = "SelfieSegmentationGpuApp", -) - -ios_application( - name = "SelfieSegmentationGpuApp", - app_icons = ["//mediapipe/examples/ios/common:AppIcon"], - bundle_id = BUNDLE_ID_PREFIX + ".SelfieSegmentationGpu", - families = [ - "iphone", - "ipad", - ], - infoplists = [ - "//mediapipe/examples/ios/common:Info.plist", - "Info.plist", - ], - minimum_os_version = MIN_IOS_VERSION, - provisioning_profile = example_provisioning(), - deps = [ - ":SelfieSegmentationGpuAppLibrary", - "@ios_opencv//:OpencvFramework", - ], -) - -objc_library( - name = "SelfieSegmentationGpuAppLibrary", - data = [ - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu.binarypb", - "//mediapipe/modules/selfie_segmentation:selfie_segmentation.tflite", - ], - deps = [ - "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", - ] + select({ - "//mediapipe:ios_i386": [], - "//mediapipe:ios_x86_64": [], - "//conditions:default": [ - "//mediapipe/graphs/selfie_segmentation:selfie_segmentation_gpu_deps", - ], - }), -) diff --git a/mediapipe/examples/ios/selfiesegmentationgpu/Info.plist b/mediapipe/examples/ios/selfiesegmentationgpu/Info.plist deleted file mode 100644 index e4349f567..000000000 --- a/mediapipe/examples/ios/selfiesegmentationgpu/Info.plist +++ /dev/null @@ -1,14 +0,0 @@ - - - - - CameraPosition - front - GraphOutputStream - output_video - GraphInputStream - input_video - GraphName - selfie_segmentation_gpu - - diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD deleted file mode 100644 index 109625bbb..000000000 --- a/mediapipe/framework/BUILD +++ /dev/null @@ -1,1677 +0,0 @@ -# -# Copyright 2019 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -package_group( - name = "mediapipe_internal", - packages = [ - "//mediapipe/...", - ], -) - -bzl_library( - name = "transitive_protos_bzl", - srcs = [ - "transitive_protos.bzl", - ], - visibility = ["//mediapipe/framework:__subpackages__"], -) - -bzl_library( - name = "encode_binary_proto_bzl", - srcs = [ - "encode_binary_proto.bzl", - ], - visibility = ["//visibility:public"], -) - -alias( - name = "encode_binary_proto", - actual = ":encode_binary_proto_bzl", - deprecation = "Use encode_binary_proto_bzl", - visibility = ["//visibility:public"], -) - -mediapipe_proto_library( - name = "calculator_proto", - srcs = ["calculator.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:mediapipe_options_proto", - "//mediapipe/framework:packet_factory_proto", - "//mediapipe/framework:packet_generator_proto", - "//mediapipe/framework:status_handler_proto", - "//mediapipe/framework:stream_handler_proto", - "@com_google_protobuf//:any_proto", - ], -) - -mediapipe_proto_library( - name = "calculator_options_proto", - srcs = ["calculator_options.proto"], - visibility = ["//visibility:public"], -) - -mediapipe_proto_library( - name = "calculator_contract_test_proto", - testonly = 1, - srcs = ["calculator_contract_test.proto"], - visibility = ["//mediapipe/framework:__subpackages__"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "calculator_profile_proto", - srcs = ["calculator_profile.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "mediapipe_options_proto", - srcs = ["mediapipe_options.proto"], - visibility = [":mediapipe_internal"], -) - -mediapipe_proto_library( - name = "packet_factory_proto", - srcs = ["packet_factory.proto"], - visibility = [":mediapipe_internal"], -) - -mediapipe_proto_library( - name = "packet_generator_proto", - srcs = ["packet_generator.proto"], - visibility = [":mediapipe_internal"], -) - -mediapipe_proto_library( - name = "packet_test_proto", - testonly = 1, - srcs = ["packet_test.proto"], - visibility = ["//mediapipe/framework:__subpackages__"], -) - -mediapipe_proto_library( - name = "status_handler_proto", - srcs = ["status_handler.proto"], - visibility = [":mediapipe_internal"], - deps = ["//mediapipe/framework:mediapipe_options_proto"], -) - -mediapipe_proto_library( - name = "stream_handler_proto", - srcs = ["stream_handler.proto"], - visibility = [":mediapipe_internal"], - deps = ["//mediapipe/framework:mediapipe_options_proto"], -) - -mediapipe_proto_library( - name = "test_calculators_proto", - testonly = 1, - srcs = ["test_calculators.proto"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_proto", - "//mediapipe/framework:calculator_proto", - ], -) - -mediapipe_proto_library( - name = "thread_pool_executor_proto", - srcs = ["thread_pool_executor.proto"], - visibility = [":mediapipe_internal"], - deps = ["//mediapipe/framework:mediapipe_options_proto"], -) - -cc_library( - name = "calculator_base", - srcs = ["calculator_base.cc"], - hdrs = ["calculator_base.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":calculator_context", - ":calculator_contract", - ":port", - ":timestamp", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "calculator_context", - srcs = ["calculator_context.cc"], - hdrs = ["calculator_context.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":calculator_state", - ":counter", - ":graph_service", - ":input_stream_shard", - ":output_stream_shard", - ":packet", - ":packet_set", - ":port", - ":timestamp", - "//mediapipe/framework/port:any_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "calculator_context_manager", - srcs = ["calculator_context_manager.cc"], - hdrs = ["calculator_context_manager.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":calculator_context", - ":calculator_state", - ":timestamp", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "calculator_contract", - srcs = ["calculator_contract.cc"], - hdrs = ["calculator_contract.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":graph_service", - ":packet_type", - ":port", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:status_handler_cc_proto", - "//mediapipe/framework:stream_handler_cc_proto", - "//mediapipe/framework/port:any_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:options_map", - "//mediapipe/framework/tool:tag_map", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "legacy_calculator_support", - srcs = ["legacy_calculator_support.cc"], - hdrs = ["legacy_calculator_support.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":calculator_context", - ":calculator_contract", - ], -) - -cc_library( - name = "calculator_framework", - hdrs = ["calculator_framework.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":calculator_base", - ":calculator_graph", - ":calculator_registry", - ":counter_factory", - ":input_stream", - ":output_side_packet", - ":output_stream", - ":packet", - ":packet_generator", - ":packet_generator_graph", - ":packet_set", - ":packet_type", - ":port", - ":status_handler", - ":subgraph", - ":timestamp", - ":validated_graph_config", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:validate", - "//mediapipe/framework/tool:validate_name", - ], -) - -cc_library( - name = "calculator_graph", - srcs = [ - "calculator_graph.cc", - "scheduler.cc", - ], - hdrs = [ - "calculator_graph.h", - "scheduler.h", - ], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":calculator_base", - ":calculator_node", - ":counter_factory", - ":delegating_executor", - ":mediapipe_profiling", - ":executor", - ":graph_output_stream", - ":graph_service", - ":graph_service_manager", - ":input_stream_manager", - ":input_stream_shard", - ":output_side_packet_impl", - ":output_stream", - ":output_stream_manager", - ":output_stream_poller", - ":output_stream_shard", - ":packet", - ":packet_generator", - ":packet_generator_graph", - ":packet_set", - ":packet_type", - ":port", - ":scheduler_queue", - ":status_handler", - ":thread_pool_executor", - ":timestamp", - ":validated_graph_config", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:calculator_profile_cc_proto", - "//mediapipe/framework:packet_factory_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:status_handler_cc_proto", - "//mediapipe/framework:thread_pool_executor_cc_proto", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:fixed_array", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/synchronization", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/profiler:graph_profiler", - "//mediapipe/framework/tool:fill_packet_set", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:tag_map", - "//mediapipe/framework/tool:validate", - "//mediapipe/framework/tool:validate_name", - "//mediapipe/gpu:graph_support", - "//mediapipe/util:cpu_util", - ] + select({ - "//conditions:default": [ - "//mediapipe/gpu:gpu_shared_data_internal", - "//mediapipe/gpu:gpu_service", - ], - "//mediapipe/gpu:disable_gpu": [], - }), -) - -cc_library( - name = "graph_service_manager", - srcs = ["graph_service_manager.cc"], - hdrs = ["graph_service_manager.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":graph_service", - "//mediapipe/framework:packet", - "@com_google_absl//absl/status", - ], -) - -cc_test( - name = "graph_service_manager_test", - srcs = ["graph_service_manager_test.cc"], - deps = [ - ":graph_service_manager", - "//mediapipe/framework:packet", - "//mediapipe/framework/port:gtest_main", - ], -) - -cc_library( - name = "calculator_node", - srcs = ["calculator_node.cc"], - hdrs = ["calculator_node.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":calculator_base", - ":calculator_context", - ":calculator_context_manager", - ":calculator_state", - ":counter_factory", - ":input_side_packet_handler", - ":input_stream_handler", - ":input_stream_manager", - ":input_stream_shard", - ":legacy_calculator_support", - ":mediapipe_profiling", - ":output_side_packet_impl", - ":output_stream_handler", - ":output_stream_manager", - ":output_stream_shard", - ":packet", - ":packet_set", - ":packet_type", - ":port", - ":timestamp", - ":validated_graph_config", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:stream_handler_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/profiler:graph_profiler", - "//mediapipe/framework/stream_handler:default_input_stream_handler", - "//mediapipe/framework/stream_handler:in_order_output_stream_handler", - "//mediapipe/framework/tool:name_util", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:tag_map", - "//mediapipe/framework/tool:validate_name", - "//mediapipe/gpu:graph_support", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "calculator_registry", - hdrs = ["calculator_registry.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":calculator_base", - ], -) - -cc_library( - name = "calculator_runner", - testonly = 1, - srcs = ["calculator_runner.cc"], - hdrs = ["calculator_runner.h"], - visibility = ["//visibility:public"], - deps = [ - ":calculator_framework", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "calculator_state", - srcs = ["calculator_state.cc"], - hdrs = ["calculator_state.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":counter", - ":counter_factory", - ":graph_service", - ":graph_service_manager", - ":input_stream", - ":output_stream", - ":packet", - ":packet_set", - ":port", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/port:any_proto", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/tool:options_map", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "camera_intrinsics", - hdrs = ["camera_intrinsics.h"], - visibility = ["//visibility:public"], -) - -cc_library( - name = "collection", - hdrs = ["collection.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":collection_item_id", - ":type_map", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/tool:tag_map", - "//mediapipe/framework/tool:tag_map_helper", - "//mediapipe/framework/tool:validate_name", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "collection_item_id", - srcs = ["collection_item_id.cc"], - hdrs = ["collection_item_id.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/deps:intops", - ], -) - -cc_library( - name = "counter", - hdrs = ["counter.h"], - visibility = ["//visibility:public"], - deps = ["//mediapipe/framework/port:integral_types"], -) - -cc_library( - name = "counter_factory", - srcs = ["counter_factory.cc"], - hdrs = ["counter_factory.h"], - visibility = ["//visibility:public"], - deps = [ - ":counter", - ":port", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:map_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - ], -) - -cc_library( - name = "delegating_executor", - srcs = ["delegating_executor.cc"], - hdrs = ["delegating_executor.h"], - visibility = ["//visibility:public"], - deps = [ - ":executor", - ], -) - -cc_library( - name = "demangle", - hdrs = ["demangle.h"], - defines = select({ - "//mediapipe/framework/profiler:android_release": [ - "MEDIAPIPE_HAS_CXA_DEMANGLE=0", - ], - "//conditions:default": [], - }), - visibility = ["//visibility:public"], -) - -cc_library( - name = "mediapipe_profiling", - hdrs = [ - "mediapipe_profiling.h", - "platform_specific_profiling.h", - ], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - "//mediapipe/framework/profiler:graph_profiler", - ], -) - -cc_library( - name = "executor", - srcs = ["executor.cc"], - hdrs = ["executor.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - ], -) - -cc_library( - name = "graph_output_stream", - srcs = ["graph_output_stream.cc"], - hdrs = ["graph_output_stream.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":input_stream_handler", - ":input_stream_manager", - ":output_stream_manager", - ":packet", - ":packet_set", - ":packet_type", - ":timestamp", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "graph_service", - hdrs = ["graph_service.h"], - visibility = [":mediapipe_internal"], - deps = [ - "@com_google_absl//absl/base:core_headers", - ], -) - -cc_library( - name = "input_side_packet_handler", - srcs = ["input_side_packet_handler.cc"], - hdrs = ["input_side_packet_handler.h"], - visibility = ["//visibility:public"], - deps = [ - ":collection_item_id", - ":packet", - ":packet_set", - ":packet_type", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:fill_packet_set", - ], -) - -cc_library( - name = "input_stream", - hdrs = ["input_stream.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":packet", - ":port", - "@com_google_absl//absl/base:core_headers", - ], -) - -cc_library( - name = "input_stream_handler", - srcs = ["input_stream_handler.cc"], - hdrs = ["input_stream_handler.h"], - visibility = [ - ":mediapipe_internal", - "//research/interaction/mediapipe/calculators:__pkg__", - ], - deps = [ - ":calculator_context", - ":calculator_context_manager", - ":collection", - ":collection_item_id", - ":input_stream_manager", - ":input_stream_shard", - ":mediapipe_profiling", - ":packet", - ":packet_set", - ":packet_type", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "input_stream_manager", - srcs = ["input_stream_manager.cc"], - hdrs = ["input_stream_manager.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":packet", - ":packet_type", - ":port", - ":timestamp", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "input_stream_shard", - srcs = ["input_stream_shard.cc"], - hdrs = ["input_stream_shard.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":input_stream", - ":packet", - ":packet_type", - ":port", - ":timestamp", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "lifetime_tracker", - testonly = 1, - hdrs = ["lifetime_tracker.h"], - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "output_side_packet", - hdrs = ["output_side_packet.h"], - visibility = ["//visibility:public"], - deps = [ - ":packet", - ], -) - -cc_library( - name = "output_side_packet_impl", - srcs = ["output_side_packet_impl.cc"], - hdrs = ["output_side_packet_impl.h"], - visibility = ["//visibility:public"], - deps = [ - ":collection_item_id", - ":input_side_packet_handler", - ":output_side_packet", - ":packet", - ":packet_type", - ":port", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "output_stream", - hdrs = ["output_stream.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":packet", - ":port", - ":timestamp", - "//mediapipe/framework/port:logging", - "@com_google_absl//absl/base:core_headers", - ], -) - -cc_library( - name = "output_stream_handler", - srcs = ["output_stream_handler.cc"], - hdrs = ["output_stream_handler.h"], - visibility = [ - ":mediapipe_internal", - ], - deps = [ - ":calculator_context_manager", - ":collection", - ":collection_item_id", - ":output_stream_manager", - ":output_stream_shard", - ":packet_set", - ":packet_type", - ":timestamp", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "output_stream_manager", - srcs = ["output_stream_manager.cc"], - hdrs = ["output_stream_manager.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":input_stream_handler", - ":output_stream_shard", - ":packet", - ":packet_type", - ":port", - ":timestamp", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "output_stream_poller", - hdrs = ["output_stream_poller.h"], - visibility = ["//visibility:public"], - deps = [ - ":graph_output_stream", - ], -) - -cc_library( - name = "output_stream_shard", - srcs = ["output_stream_shard.cc"], - hdrs = ["output_stream_shard.h"], - visibility = [":mediapipe_internal"], - deps = [ - ":output_stream", - ":packet", - ":packet_type", - ":port", - ":timestamp", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -# Defines Packet, a data carrier used throughout the framework. -cc_library( - name = "packet", - srcs = ["packet.cc"], - hdrs = ["packet.h"], - visibility = ["//visibility:public"], - deps = [ - ":port", - ":timestamp", - ":type_map", - "//mediapipe/framework/deps:no_destructor", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework/tool:type_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "packet_generator", - hdrs = ["packet_generator.h"], - visibility = ["//visibility:public"], - deps = [ - ":packet", - ":packet_set", - ":packet_type", - ":port", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "packet_generator_graph", - srcs = ["packet_generator_graph.cc"], - hdrs = ["packet_generator_graph.h"], - visibility = ["//visibility:public"], - deps = [ - ":delegating_executor", - ":executor", - ":packet", - ":packet_generator", - ":packet_type", - ":port", - ":thread_pool_executor", - ":validated_graph_config", - "//mediapipe/framework:packet_factory_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "packet_set", - hdrs = ["packet_set.h"], - visibility = ["//visibility:public"], - deps = [ - ":collection", - ":packet", - ], -) - -cc_library( - name = "packet_type", - srcs = ["packet_type.cc"], - hdrs = ["packet_type.h"], - visibility = ["//visibility:public"], - deps = [ - ":collection", - ":packet", - ":packet_set", - ":type_map", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:validate_name", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - ], -) - -# When --copt=-fno-rtti is set, MEDIAPIPE_HAS_RTTI is cleared in port.h. -# To explicitly clear MEDIAPIPE_HAS_RTTI, compile with: -# bazel build --define=disable_rtti_and_exceptions=true -config_setting( - name = "disable_rtti_and_exceptions", - define_values = {"disable_rtti_and_exceptions": "true"}, - visibility = ["//visibility:public"], -) - -cc_library( - name = "port", - hdrs = ["port.h"], - defines = select({ - "//conditions:default": [], - }) + select({ - "//conditions:default": [], - "//mediapipe/gpu:disable_gpu": ["MEDIAPIPE_DISABLE_GPU=1"], - }) + select({ - "//conditions:default": [], - "//mediapipe/framework:disable_rtti_and_exceptions": [ - "MEDIAPIPE_HAS_RTTI=0", - ], - }), - visibility = [ - "//mediapipe/calculators:__subpackages__", - "//mediapipe/framework:__subpackages__", - "//mediapipe/framework/port:__pkg__", - "//mediapipe/gpu:__pkg__", - "//mediapipe/util:__subpackages__", - ], -) - -cc_library( - name = "scheduler_queue", - srcs = ["scheduler_queue.cc"], - hdrs = [ - "scheduler_queue.h", - "scheduler_shared.h", - ], - copts = select({ - "//conditions:default": [], - "//mediapipe:apple": [ - "-ObjC++", - ], - }), - visibility = [":mediapipe_internal"], - deps = [ - ":calculator_context", - ":calculator_node", - ":executor", - "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/synchronization", - ], -) - -cc_library( - name = "status_handler", - hdrs = ["status_handler.h"], - visibility = ["//visibility:public"], - deps = [ - ":packet_set", - ":packet_type", - ":port", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/memory", - ], -) - -cc_library( - name = "subgraph", - srcs = ["subgraph.cc"], - hdrs = ["subgraph.h"], - visibility = ["//visibility:public"], - deps = [ - ":graph_service", - ":graph_service_manager", - ":port", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/deps:registration", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework/tool:calculator_graph_template_cc_proto", - "//mediapipe/framework/tool:options_util", - "//mediapipe/framework/tool:template_expander", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", - "@com_google_absl//absl/types:optional", - ], -) - -cc_library( - name = "test_calculators", - testonly = 1, - srcs = ["test_calculators.cc"], - visibility = ["//visibility:public"], - deps = [ - ":calculator_framework", - "//mediapipe/framework:test_calculators_cc_proto", - "//mediapipe/framework/deps:mathutil", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@eigen_archive//:eigen3", - ], - alwayslink = 1, -) - -cc_library( - name = "test_service", - testonly = 1, - srcs = ["test_service.cc"], - hdrs = ["test_service.h"], - visibility = ["//visibility:public"], - deps = [ - ":calculator_contract", - ":calculator_framework", - ":graph_service", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "thread_pool_executor", - srcs = ["thread_pool_executor.cc"], - hdrs = ["thread_pool_executor.h"], - visibility = ["//visibility:public"], - deps = [ - ":executor", - "//mediapipe/framework:thread_pool_executor_cc_proto", - "//mediapipe/framework/deps:thread_options", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:statusor", - "//mediapipe/framework/port:threadpool", - "//mediapipe/util:cpu_util", - ], -) - -cc_library( - name = "timestamp", - srcs = ["timestamp.cc"], - hdrs = ["timestamp.h"], - visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework/deps:intops", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "throttler", - hdrs = ["throttler.h"], - visibility = ["//visibility:public"], -) - -cc_library( - name = "type_map", - hdrs = ["type_map.h"], - visibility = ["//visibility:public"], - deps = [ - ":demangle", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:type_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/synchronization", - ], - alwayslink = 1, -) - -cc_library( - name = "basic_types_registration", - srcs = ["basic_types_registration.cc"], - visibility = ["//visibility:public"], - deps = [ - ":type_map", - "//mediapipe/framework/port:integral_types", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - -cc_library( - name = "validated_graph_config", - srcs = ["validated_graph_config.cc"], - hdrs = ["validated_graph_config.h"], - visibility = ["//visibility:public"], - deps = [ - ":calculator_base", - ":calculator_contract", - ":graph_service_manager", - ":legacy_calculator_support", - ":packet", - ":packet_generator", - ":packet_set", - ":packet_type", - ":port", - ":status_handler", - ":subgraph", - ":timestamp", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:status_handler_cc_proto", - "//mediapipe/framework:stream_handler_cc_proto", - "//mediapipe/framework:thread_pool_executor_cc_proto", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:map_util", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:source_location", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:topologicalsorter", - "//mediapipe/framework/tool:name_util", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:subgraph_expansion", - "//mediapipe/framework/tool:validate", - "//mediapipe/framework/tool:validate_name", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "validated_graph_config_test", - srcs = ["validated_graph_config_test.cc"], - deps = [ - ":calculator_framework", - ":graph_service", - ":graph_service_manager", - ":validated_graph_config", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/api2:node", - "//mediapipe/framework/api2:port", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "graph_validation", - hdrs = ["graph_validation.h"], - visibility = ["//visibility:public"], - deps = [ - ":calculator_framework", - "//mediapipe/framework/port:status", - ], -) - -# cc tests -cc_test( - name = "calculator_base_test", - size = "medium", - srcs = ["calculator_base_test.cc"], - linkstatic = 1, - deps = [ - ":calculator_base", - ":calculator_context", - ":calculator_context_manager", - ":calculator_registry", - ":calculator_state", - ":output_stream", - ":output_stream_manager", - ":output_stream_shard", - ":packet_set", - ":packet_type", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:tag_map_helper", - "@com_google_absl//absl/container:flat_hash_set", - ], -) - -cc_test( - name = "calculator_contract_test", - srcs = ["calculator_contract_test.cc"], - linkstatic = 1, - deps = [ - ":calculator_contract", - ":calculator_contract_test_cc_proto", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:status_handler_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_test( - name = "calculator_node_test", - size = "small", - srcs = ["calculator_node_test.cc"], - linkstatic = 1, - deps = [ - ":calculator_framework", - ":calculator_node", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:source", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "calculator_graph_event_loop_test", - size = "small", - srcs = ["calculator_graph_event_loop_test.cc"], - deps = [ - ":calculator_framework", - ":calculator_graph", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "calculator_graph_stopping_test", - size = "small", - srcs = ["calculator_graph_stopping_test.cc"], - deps = [ - ":calculator_framework", - ":calculator_graph", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "calculator_parallel_execution_test", - srcs = ["calculator_parallel_execution_test.cc"], - deps = [ - ":calculator_framework", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "calculator_runner_test", - size = "medium", - srcs = ["calculator_runner_test.cc"], - deps = [ - ":calculator_base", - ":calculator_registry", - ":calculator_runner", - ":input_stream", - ":output_stream", - ":packet_type", - ":timestamp", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "calculator_context_test", - size = "medium", - srcs = ["calculator_context_test.cc"], - linkstatic = 1, - deps = [ - ":calculator_context", - ":calculator_context_manager", - ":calculator_state", - ":output_stream", - ":output_stream_manager", - ":output_stream_shard", - ":packet_set", - ":packet_type", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/testdata:night_light_calculator_cc_proto", - "//mediapipe/framework/testdata:sky_light_calculator_cc_proto", - "//mediapipe/framework/tool:status_util", - "//mediapipe/framework/tool:tag_map_helper", - ], -) - -cc_test( - name = "calculator_graph_test", - size = "small", - srcs = [ - "calculator_graph_test.cc", - ], - linkstatic = 1, - visibility = ["//visibility:public"], - deps = [ - ":calculator_framework", - ":calculator_graph", - ":collection_item_id", - ":counter_factory", - ":executor", - ":input_stream_handler", - ":lifetime_tracker", - ":output_stream_poller", - ":packet_set", - ":packet_type", - ":status_handler", - ":subgraph", - ":test_calculators", - ":thread_pool_executor", - ":timestamp", - ":type_map", - "//mediapipe/calculators/core:counting_source_calculator", - "//mediapipe/calculators/core:mux_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework:thread_pool_executor_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:barrier_input_stream_handler", - "//mediapipe/framework/stream_handler:early_close_input_stream_handler", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/stream_handler:mux_input_stream_handler", - "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", - "//mediapipe/framework/stream_handler:timestamp_align_input_stream_handler", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool:status_util", - "@com_google_absl//absl/container:fixed_array", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "calculator_graph_bounds_test", - size = "small", - srcs = [ - "calculator_graph_bounds_test.cc", - ], - visibility = ["//visibility:public"], - deps = [ - ":calculator_context", - ":calculator_framework", - ":test_calculators", - ":thread_pool_executor", - ":timestamp", - ":type_map", - "//mediapipe/calculators/core:counting_source_calculator", - "//mediapipe/calculators/core:mux_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - "//mediapipe/framework/stream_handler:barrier_input_stream_handler", - "//mediapipe/framework/stream_handler:early_close_input_stream_handler", - "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", - "//mediapipe/framework/stream_handler:immediate_input_stream_handler", - "//mediapipe/framework/stream_handler:mux_input_stream_handler", - "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "calculator_graph_side_packet_test", - size = "small", - srcs = [ - "calculator_graph_side_packet_test.cc", - ], - visibility = ["//visibility:public"], - deps = [ - ":calculator_framework", - ":test_calculators", - "//mediapipe/calculators/core:counting_source_calculator", - "//mediapipe/calculators/core:mux_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - "@com_google_absl//absl/time", - ], -) - -cc_test( - name = "collection_test", - size = "small", - srcs = ["collection_test.cc"], - linkstatic = 1, - deps = [ - ":collection", - ":packet_set", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:tag_map_helper", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "graph_service_test", - size = "small", - srcs = [ - "graph_service_test.cc", - ], - visibility = ["//visibility:public"], - deps = [ - ":calculator_contract", - ":calculator_framework", - ":graph_service", - ":test_service", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "//mediapipe/framework/tool:sink", - ], -) - -cc_test( - name = "input_stream_manager_test", - size = "small", - srcs = ["input_stream_manager_test.cc"], - linkstatic = 1, - deps = [ - ":input_stream_manager", - ":input_stream_shard", - ":lifetime_tracker", - ":packet", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "output_stream_manager_test", - size = "small", - srcs = ["output_stream_manager_test.cc"], - linkstatic = 1, - deps = [ - ":input_stream_handler", - ":input_stream_manager", - ":output_stream_manager", - ":output_stream_shard", - ":packet", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/stream_handler:default_input_stream_handler", - "//mediapipe/framework/tool:tag_map_helper", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "packet_delete_test", - size = "small", - srcs = ["packet_delete_test.cc"], - copts = [ - "-Werror", - ], - linkstatic = 1, - deps = [ - ":packet", - "//mediapipe/framework/port:gtest_main", - ], -) - -cc_test( - name = "executor_external_build_test", - size = "small", - srcs = ["executor_external_build_test.cc"], - linkstatic = 1, - deps = [ - ":executor", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:threadpool", - "@com_google_absl//absl/memory", - ], -) - -cc_test( - name = "packet_test", - size = "medium", - srcs = ["packet_test.cc"], - linkstatic = 1, - deps = [ - ":packet", - ":packet_test_cc_proto", - ":type_map", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "packet_registration_test", - size = "small", - srcs = ["packet_registration_test.cc"], - deps = [ - ":calculator_framework", - ":packet", - ":packet_test_cc_proto", - ":type_map", - "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "packet_generator_test", - size = "small", - srcs = ["packet_generator_test.cc"], - deps = [ - ":packet_generator", - ":packet_type", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/tool:validate_type", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "timestamp_test", - size = "small", - srcs = ["timestamp_test.cc"], - linkstatic = 1, - deps = [ - ":timestamp", - "//mediapipe/framework/port:gtest_main", - ], -) - -cc_test( - name = "graph_validation_test", - srcs = ["graph_validation_test.cc"], - deps = [ - ":calculator_contract_test_cc_proto", - ":calculator_framework", - ":graph_validation", - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/core:default_side_packet_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework:packet_generator_cc_proto", - "//mediapipe/framework:status_handler_cc_proto", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:template_parser", - ], -) - -cc_test( - name = "subgraph_test", - srcs = ["subgraph_test.cc"], - deps = [ - ":calculator_framework", - ":graph_service_manager", - ":subgraph", - ":test_calculators", - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/core:pass_through_calculator", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:sink", - "//mediapipe/framework/tool/testdata:dub_quad_test_subgraph", - "@com_google_absl//absl/strings:str_format", - ], -) - -# Expose the proto source files for building mediapipe AAR. -filegroup( - name = "protos_src", - srcs = glob(["*.proto"]), - visibility = ["//mediapipe:__subpackages__"], -) diff --git a/mediapipe/framework/api2/BUILD b/mediapipe/framework/api2/BUILD deleted file mode 100644 index 1a4faca13..000000000 --- a/mediapipe/framework/api2/BUILD +++ /dev/null @@ -1,220 +0,0 @@ -package( - default_visibility = ["//visibility:public"], - features = ["-use_header_modules"], -) - -licenses(["notice"]) - -cc_library( - name = "const_str", - hdrs = ["const_str.h"], -) - -cc_library( - name = "builder", - hdrs = ["builder.h"], - deps = [ - ":const_str", - ":contract", - ":node", - ":packet", - ":port", - "//mediapipe/framework:calculator_base", - "//mediapipe/framework:calculator_contract", - "@com_google_absl//absl/container:flat_hash_map", - ], -) - -cc_test( - name = "builder_test", - srcs = ["builder_test.cc"], - deps = [ - ":builder", - ":node", - ":packet", - ":port", - ":tag", - ":test_contracts", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "contract", - hdrs = ["contract.h"], - deps = [ - ":const_str", - ":packet", - ":port", - ":tag", - ":tuple", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:output_side_packet", - "//mediapipe/framework/port:logging", - ], -) - -cc_test( - name = "contract_test", - srcs = ["contract_test.cc"], - deps = [ - ":contract", - ":port", - ":tag", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - ], -) - -cc_library( - name = "node", - srcs = ["node.cc"], - hdrs = ["node.h"], - deps = [ - ":const_str", - ":contract", - ":packet", - ":port", - "//mediapipe/framework:calculator_base", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:subgraph", - "//mediapipe/framework/deps:no_destructor", - ], -) - -cc_library( - name = "test_contracts", - testonly = 1, - hdrs = ["test_contracts.h"], - deps = [ - ":node", - ], -) - -cc_test( - name = "node_test", - srcs = ["node_test.cc"], - deps = [ - ":node", - ":packet", - ":port", - ":test_contracts", - ":tuple", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", - ], -) - -cc_library( - name = "packet", - srcs = ["packet.cc"], - hdrs = ["packet.h"], - deps = [ - ":tuple", - "//mediapipe/framework:packet", - "//mediapipe/framework/port:logging", - "@com_google_absl//absl/meta:type_traits", - ], -) - -cc_test( - name = "packet_test", - size = "small", - srcs = [ - "packet_test.cc", - ], - deps = [ - ":packet", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "port", - hdrs = ["port.h"], - deps = [ - ":const_str", - ":packet", - "//mediapipe/framework:calculator_context", - "//mediapipe/framework:calculator_contract", - "//mediapipe/framework:output_side_packet", - "//mediapipe/framework/port:logging", - "@com_google_absl//absl/strings", - ], -) - -cc_test( - name = "port_test", - size = "small", - srcs = [ - "port_test.cc", - ], - deps = [ - ":port", - "//mediapipe/framework/port:gtest_main", - ], -) - -cc_test( - name = "subgraph_test", - srcs = ["subgraph_test.cc"], - deps = [ - ":builder", - ":node", - ":packet", - ":port", - ":test_contracts", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/deps:message_matchers", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/tool:subgraph_expansion", - ], -) - -cc_library( - name = "tag", - hdrs = ["tag.h"], - deps = [":const_str"], -) - -cc_test( - name = "tag_test", - size = "small", - srcs = [ - "tag_test.cc", - ], - deps = [ - ":tag", - "//mediapipe/framework/port:gtest_main", - ], -) - -cc_library( - name = "tuple", - hdrs = ["tuple.h"], - deps = ["@com_google_absl//absl/meta:type_traits"], -) - -cc_test( - name = "tuple_test", - size = "small", - srcs = [ - "tuple_test.cc", - ], - deps = [ - ":tuple", - "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/strings", - ], -) diff --git a/mediapipe/framework/api2/README.md b/mediapipe/framework/api2/README.md deleted file mode 100644 index eb53dd67e..000000000 --- a/mediapipe/framework/api2/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# New MediaPipe APIs - -This directory defines new APIs for MediaPipe: - -- Node API, an update to the Calculator API for defining MediaPipe components. -- Builder API, for assembling CalculatorGraphConfigs with C++, as an alternative - to using the proto API directly. - -The new APIs interoperate fully with the existing framework code, and we are -adopting them in our calculators. We are still making improvements, and the -placement of this code under the `mediapipe::api2` namespace is not final. - -Developers are welcome to try out these APIs as early adopters, but there may be -breaking changes. - -## Node API - -This API can be used to define calculators. It is designed to be more type-safe -and less verbose than the original API. - -Input/output ports (streams and side packets) can now be declared as typed -constants, instead of using plain strings for access. - -For example, instead of - -``` -constexpr char kSelectTag[] = "SELECT"; -if (cc->Inputs().HasTag(kSelectTag)) { - cc->Inputs().Tag(kSelectTag).Set(); -} -``` - -you can write - -``` -static constexpr Input::Optional kSelect{"SELECT"}; -``` - -Instead of setting up the contract procedurally in `GetContract`, add ports to -the contract declaratively, as follows: - -``` -MEDIAPIPE_NODE_CONTRACT(kInput, kOutput); -``` - -To access an input in Process, instead of - -``` -int select = cc->Inputs().Tag(kSelectTag).Get(); -``` - -write - -``` -int select = kSelectTag(cc).Get(); // alternative: *kSelectTag(cc) -``` - -Sets of multiple ports can be declared with `::Multiple`. Note, also, that a tag -string must always be provided when declaring a port; use `""` for untagged -ports. For example: - - -``` -for (int i = 0; i < cc->Inputs().NumEntries(); ++i) { - cc->Inputs().Index(i).SetAny(); -} -``` - -becomes - -``` -static constexpr Input::Multiple kIn{""}; -``` - -For output ports, the payload can be passed directly to the `Send` method. For -example, instead of - -``` -cc->Outputs().Index(0).Add( - new std::pair(cc->Inputs().Index(0).Value(), - cc->Inputs().Index(1).Value()), - cc->InputTimestamp()); -``` - -you can write - -``` -kPair(cc).Send({kIn(cc)[0].packet(), kIn(cc)[1].packet()}); -``` - -The input timestamp is propagated to the outputs by default. If your calculator -wants to alter timestamps, it must add a `TimestampChange` entry to its contract -declaration. For example: - -``` -MEDIAPIPE_NODE_CONTRACT(kMain, kLoop, kPrevLoop, - StreamHandler("ImmediateInputStreamHandler"), - TimestampChange::Arbitrary()); -``` - -Several calculators in -[`calculators/core`](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core) and -[`calculators/tensor`](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/tensor) -have been updated to use this API. Reference them for more examples. - -More complete documentation will be provided in the future. - -## Builder API - -Documentation will be provided in the future. diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h deleted file mode 100644 index ae32c628a..000000000 --- a/mediapipe/framework/api2/builder.h +++ /dev/null @@ -1,575 +0,0 @@ -#ifndef MEDIAPIPE_FRAMEWORK_API2_BUILDER_H_ -#define MEDIAPIPE_FRAMEWORK_API2_BUILDER_H_ - -#include -#include - -#include "absl/container/flat_hash_map.h" -#include "mediapipe/framework/api2/const_str.h" -#include "mediapipe/framework/api2/contract.h" -#include "mediapipe/framework/api2/node.h" -#include "mediapipe/framework/api2/packet.h" -#include "mediapipe/framework/api2/port.h" -#include "mediapipe/framework/calculator_base.h" -#include "mediapipe/framework/calculator_contract.h" - -namespace mediapipe { -namespace api2 { -namespace builder { - -template -T& GetWithAutoGrow(std::vector>* vecp, int index) { - auto& vec = *vecp; - if (vec.size() <= index) { - vec.resize(index + 1); - } - if (vec[index] == nullptr) { - vec[index] = absl::make_unique(); - } - return *vec[index]; -} - -struct TagIndexLocation { - const std::string& tag; - std::size_t index; - std::size_t count; -}; - -template -class TagIndexMap { - public: - std::vector>& operator[](const std::string& tag) { - return map_[tag]; - } - - void Visit(std::function fun) const { - for (const auto& tagged : map_) { - TagIndexLocation loc{tagged.first, 0, tagged.second.size()}; - for (const auto& item : tagged.second) { - fun(loc, *item); - ++loc.index; - } - } - } - - void Visit(std::function fun) { - for (auto& tagged : map_) { - TagIndexLocation loc{tagged.first, 0, tagged.second.size()}; - for (auto& item : tagged.second) { - fun(loc, item.get()); - ++loc.index; - } - } - } - - // Note: entries are held by a unique_ptr to ensure pointers remain valid. - // Should use absl::flat_hash_map but ordering keys for now. - std::map>> map_; -}; - -// These structs are used internally to store information about the endpoints -// of a connection. -struct SourceBase; -struct DestinationBase { - SourceBase* source = nullptr; -}; -struct SourceBase { - std::vector dests_; - std::string name_; -}; - -// Following existing GraphConfig usage, we allow using a multiport as a single -// port as well. This is necessary for generic nodes, since we have no -// information about which ports are meant to be multiports or not, but it is -// also convenient with typed nodes. -template -class MultiPort : public Single { - public: - using Base = typename Single::Base; - - explicit MultiPort(std::vector>* vec) - : Single(vec), vec_(*vec) {} - - Single operator[](int index) { - CHECK_GE(index, 0); - return Single{&GetWithAutoGrow(&vec_, index)}; - } - - private: - std::vector>& vec_; -}; - -// These classes wrap references to the underlying source/destination -// endpoints, adding type information and the user-visible API. -template -class DestinationImpl { - public: - using Base = DestinationBase; - - explicit DestinationImpl(std::vector>* vec) - : DestinationImpl(&GetWithAutoGrow(vec, 0)) {} - explicit DestinationImpl(DestinationBase* base) : base_(*base) {} - DestinationBase& base_; -}; - -template -class DestinationImpl - : public MultiPort> { - public: - using MultiPort>::MultiPort; -}; - -template -class SourceImpl { - public: - using Base = SourceBase; - - // Src is used as the return type of fluent methods below. Since these are - // single-port methods, it is desirable to always decay to a reference to the - // single-port superclass, even if they are called on a multiport. - using Src = SourceImpl; - template - using Dst = DestinationImpl; - - // clang-format off - template - struct AllowConnection : public std::integral_constant{} || std::is_same{} || - std::is_same{}> {}; - // clang-format on - - explicit SourceImpl(std::vector>* vec) - : SourceImpl(&GetWithAutoGrow(vec, 0)) {} - explicit SourceImpl(SourceBase* base) : base_(*base) {} - - template {}, int>::type = 0> - Src& AddTarget(const Dst& dest) { - CHECK(dest.base_.source == nullptr); - dest.base_.source = &base_; - base_.dests_.emplace_back(&dest.base_); - return *this; - } - Src& SetName(std::string name) { - base_.name_ = std::move(name); - return *this; - } - template - Src& operator>>(const Dst& dest) { - return AddTarget(dest); - } - - private: - SourceBase& base_; -}; - -template -class SourceImpl - : public MultiPort> { - public: - using MultiPort>::MultiPort; -}; - -// A source and a destination correspond to an output/input stream on a node, -// and a side source and side destination correspond to an output/input side -// packet. -// For graph inputs/outputs, however, the inputs are sources, and the outputs -// are destinations. This is because graph ports are connected "from inside" -// when building the graph. -template -using Source = SourceImpl; -template -using SideSource = SourceImpl; -template -using Destination = DestinationImpl; -template -using SideDestination = DestinationImpl; - -class NodeBase { - public: - // TODO: right now access to an indexed port is made directly by - // specifying both a tag and an index. It would be better to represent this - // as a two-step lookup, first getting a multi-port, and then accessing one - // of its entries by index. However, for nodes without visible contracts we - // can't know whether a tag is indexable or not, so we would need the - // multi-port to also be usable as a port directly (representing index 0). - Source Out(const std::string& tag) { - return Source(&out_streams_[tag]); - } - - Destination In(const std::string& tag) { - return Destination(&in_streams_[tag]); - } - - SideSource SideOut(const std::string& tag) { - return SideSource(&out_sides_[tag]); - } - - SideDestination SideIn(const std::string& tag) { - return SideDestination(&in_sides_[tag]); - } - - // Convenience methods for accessing purely index-based ports. - Source Out(int index) { return Out("")[index]; } - - Destination In(int index) { return In("")[index]; } - - SideSource SideOut(int index) { return SideOut("")[index]; } - - SideDestination SideIn(int index) { return SideIn("")[index]; } - - template - T& GetOptions() { - options_used_ = true; - return *options_.MutableExtension(T::ext); - } - - protected: - NodeBase(std::string type) : type_(std::move(type)) {} - - std::string type_; - TagIndexMap in_streams_; - TagIndexMap out_streams_; - TagIndexMap in_sides_; - TagIndexMap out_sides_; - CalculatorOptions options_; - // ideally we'd just check if any extensions are set on options_ - bool options_used_ = false; - friend class Graph; -}; - -template -class Node; -#if __cplusplus >= 201703L -// Deduction guide to silence -Wctad-maybe-unsupported. -explicit Node()->Node; -#endif // C++17 - -template <> -class Node : public NodeBase { - public: - Node(std::string type) : NodeBase(std::move(type)) {} -}; - -using GenericNode = Node; - -template