From a3d36eee32999290c1461d469dc33cf63ccdf03a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 6 Apr 2020 19:10:12 -0700 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: 53a42bf7ad836321123cb7b6c80b0f2e13fbf83e --- .bazelrc | 10 +- Dockerfile | 2 +- WORKSPACE | 168 +- mediapipe/BUILD | 3 + mediapipe/calculators/core/BUILD | 3 + .../calculators/core/begin_loop_calculator.cc | 5 + .../calculators/core/end_loop_calculator.cc | 4 + .../core/split_vector_calculator.cc | 4 + mediapipe/calculators/image/BUILD | 57 + .../image/feature_detector_calculator.cc | 210 ++ .../image/feature_detector_calculator.proto | 24 + .../image/image_cropping_calculator.cc | 12 +- ...encoded_image_to_image_frame_calculator.cc | 30 +- ...oded_image_to_image_frame_calculator.proto | 30 + ...low_session_from_saved_model_calculator.cc | 10 +- ..._session_from_saved_model_calculator.proto | 4 +- ...flow_session_from_saved_model_generator.cc | 10 +- ...w_session_from_saved_model_generator.proto | 4 +- mediapipe/calculators/tflite/BUILD | 2 +- .../tflite/tflite_inference_calculator.cc | 2 +- .../tflite_tensors_to_landmarks_calculator.cc | 2 +- mediapipe/calculators/tflite/util.h | 10 +- mediapipe/calculators/util/BUILD | 8 +- .../util/annotation_overlay_calculator.cc | 16 +- .../util/labels_to_render_data_calculator.cc | 17 +- .../util/local_file_contents_calculator.cc | 64 +- ...imed_box_list_to_render_data_calculator.cc | 10 +- mediapipe/calculators/video/BUILD | 85 + .../video/box_detector_calculator.cc | 393 +++ .../video/box_detector_calculator.proto | 31 + .../tracked_detection_manager_calculator.cc | 11 + ...tracked_detection_manager_calculator.proto | 28 + mediapipe/docs/face_detection_desktop.md | 2 +- mediapipe/docs/gpu.md | 4 +- mediapipe/docs/hair_segmentation_desktop.md | 2 +- mediapipe/docs/hand_tracking_desktop.md | 2 +- mediapipe/docs/install.md | 43 +- mediapipe/docs/media_sequence.md | 6 +- mediapipe/docs/mediapipe_ios_setup.md | 4 + mediapipe/docs/multi_hand_tracking_desktop.md | 2 +- .../mediapipe/apps/edgedetectiongpu/BUILD | 4 +- .../mediapipe/apps/facedetectioncpu/BUILD | 8 +- .../mediapipe/apps/facedetectiongpu/BUILD | 8 +- .../mediapipe/apps/hairsegmentationgpu/BUILD | 8 +- .../mediapipe/apps/handdetectiongpu/BUILD | 8 +- .../mediapipe/apps/handtrackinggpu/BUILD | 8 +- .../mediapipe/apps/multihandtrackinggpu/BUILD | 8 +- .../mediapipe/apps/objectdetection3d/BUILD | 8 +- .../mediapipe/apps/objectdetectioncpu/BUILD | 8 +- .../mediapipe/apps/objectdetectiongpu/BUILD | 8 +- .../mediapipe/apps/objecttrackinggpu/BUILD | 6 +- mediapipe/examples/coral/Dockerfile | 8 +- mediapipe/examples/coral/WORKSPACE | 356 -- mediapipe/examples/coral/WORKSPACE.coral | 30 + mediapipe/examples/coral/setup.sh | 15 +- .../calculators/shot_boundary_calculator.cc | 2 +- mediapipe/framework/BUILD | 4 + mediapipe/framework/calculator_graph_test.cc | 64 + mediapipe/framework/calculator_node.cc | 6 +- mediapipe/framework/calculator_profile.proto | 6 +- mediapipe/framework/formats/BUILD | 109 +- mediapipe/framework/formats/annotation/BUILD | 28 +- mediapipe/framework/input_stream_handler.cc | 32 + mediapipe/framework/input_stream_handler.h | 3 + mediapipe/framework/input_stream_manager.cc | 28 +- mediapipe/framework/output_stream_manager.cc | 4 +- mediapipe/framework/packet.h | 28 +- mediapipe/framework/port/BUILD | 1 + mediapipe/framework/port/build_config.bzl | 100 +- mediapipe/framework/port/opencv_video_inc.h | 3 +- mediapipe/framework/profiler/BUILD | 6 +- .../framework/profiler/graph_profiler_stub.h | 3 + mediapipe/framework/profiler/graph_tracer.cc | 13 +- mediapipe/framework/profiler/graph_tracer.h | 6 +- .../framework/profiler/graph_tracer_test.cc | 399 ++- ...apple.cc => profiler_resource_util_ios.cc} | 0 .../profiler/testdata/profile_opencv_0.pbtxt | 462 +-- .../profiler/testdata/profile_opencv_1.pbtxt | 260 +- mediapipe/framework/profiler/trace_buffer.h | 111 +- mediapipe/framework/profiler/trace_builder.cc | 113 +- mediapipe/framework/profiler/trace_builder.h | 3 + .../default_input_stream_handler.cc | 11 + .../default_input_stream_handler.h | 7 + .../immediate_input_stream_handler.cc | 24 + mediapipe/framework/subgraph.cc | 8 + mediapipe/framework/subgraph.h | 4 + mediapipe/gpu/BUILD | 19 +- mediapipe/gpu/gl_context.cc | 104 +- mediapipe/gpu/gl_context.h | 3 + mediapipe/gpu/gl_texture_buffer_pool.cc | 48 +- mediapipe/gpu/gl_texture_buffer_pool.h | 3 +- mediapipe/gpu/pixel_buffer_pool_util.mm | 32 +- .../com/google/mediapipe/components/BUILD | 16 +- .../java/com/google/mediapipe/framework/BUILD | 16 +- .../java/com/google/mediapipe/glutil/BUILD | 8 +- mediapipe/models/README.md | 17 +- mediapipe/objc/BUILD | 7 +- mediapipe/objc/MPPGraph.mm | 7 +- mediapipe/objc/util.cc | 16 +- mediapipe/util/BUILD | 6 +- mediapipe/util/sequence/media_sequence.h | 9 + .../util/sequence/media_sequence_test.cc | 27 + mediapipe/util/tracking/BUILD | 55 + mediapipe/util/tracking/box_detector.cc | 749 ++++ mediapipe/util/tracking/box_detector.h | 161 + mediapipe/util/tracking/box_detector.proto | 97 + mediapipe/util/tracking/flow_packager.cc | 27 +- mediapipe/util/tracking/image_util.cc | 2 +- mediapipe/util/tracking/motion_analysis.cc | 6 +- .../util/tracking/region_flow_computation.cc | 8 +- .../tracking/region_flow_computation_test.cc | 8 +- .../tracking/region_flow_visualization.cc | 8 +- mediapipe/util/tracking/tone_estimation.cc | 2 +- mediapipe/util/tracking/tracked_detection.cc | 14 +- mediapipe/util/tracking/tracked_detection.h | 3 +- .../tracking/tracked_detection_manager.cc | 8 +- .../util/tracking/tracked_detection_manager.h | 7 + .../tracked_detection_manager_config.proto | 28 + .../tracking_visualization_utilities.cc | 4 +- setup_opencv.sh | 104 +- ..._rules_apple_bypass_test_runner_check.diff | 13 + third_party/opencv_linux.BUILD | 3 +- ...e22eae8bf3206189a066032c66e9e5c9b4a61.diff | 65 - ...6366bcadab23a25c773b3ed405bac8ded4d0d.diff | 112 - ...31e324c8de6b52f752a39cb161d99d853ca99.diff | 3083 ----------------- .../org_tensorflow_compatibility_fixes.diff | 25 + .../org_tensorflow_protobuf_updates.diff | 90 + 127 files changed, 3910 insertions(+), 4783 deletions(-) create mode 100644 mediapipe/calculators/image/feature_detector_calculator.cc create mode 100644 mediapipe/calculators/image/feature_detector_calculator.proto create mode 100644 mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto create mode 100644 mediapipe/calculators/video/box_detector_calculator.cc create mode 100644 mediapipe/calculators/video/box_detector_calculator.proto create mode 100644 mediapipe/calculators/video/tracked_detection_manager_calculator.proto delete mode 100644 mediapipe/examples/coral/WORKSPACE create mode 100644 mediapipe/examples/coral/WORKSPACE.coral rename mediapipe/framework/profiler/{profiler_resource_util_apple.cc => profiler_resource_util_ios.cc} (100%) create mode 100644 mediapipe/util/tracking/box_detector.cc create mode 100644 mediapipe/util/tracking/box_detector.h create mode 100644 mediapipe/util/tracking/box_detector.proto create mode 100644 mediapipe/util/tracking/tracked_detection_manager_config.proto create mode 100644 third_party/build_bazel_rules_apple_bypass_test_runner_check.diff delete mode 100644 third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff delete mode 100644 third_party/org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff delete mode 100644 third_party/org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff create mode 100644 third_party/org_tensorflow_compatibility_fixes.diff create mode 100644 third_party/org_tensorflow_protobuf_updates.diff diff --git a/.bazelrc b/.bazelrc index e80bd5d7a..9c3c05f76 100644 --- a/.bazelrc +++ b/.bazelrc @@ -12,9 +12,9 @@ build --copt='-Wno-comment' build --copt='-Wno-return-type' build --copt='-Wno-unused-local-typedefs' build --copt='-Wno-ignored-attributes' -# Temporarily set the incompatibility flag for Bazel 0.27.0 and above -build --incompatible_disable_deprecated_attr_params=false -build --incompatible_depset_is_not_iterable=false + +# Tensorflow needs remote repo +build --experimental_repo_remote_exec # Sets the default Apple platform to macOS. build --apple_platform_type=macos @@ -23,6 +23,10 @@ build --apple_platform_type=macos build --apple_generate_dsym # Android configs. +# Note: the documentation tells us to use @androidndk//:default_crosstool, but +# the automatic configuration transition uses //external:android/crosstool. +# Using it here works and spares us from having two different config_settings +# for Android. build:android --crosstool_top=//external:android/crosstool build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain build:android --linkopt=-landroid diff --git a/Dockerfile b/Dockerfile index ef7b4d43a..9bb2ad1e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN pip install future RUN pip3 install six # Install bazel -ARG BAZEL_VERSION=1.1.0 +ARG BAZEL_VERSION=2.0.0 RUN mkdir /bazel && \ wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\ azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ diff --git a/WORKSPACE b/WORKSPACE index eb2b07c4d..09475b568 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,16 +2,15 @@ workspace(name = "mediapipe") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -skylib_version = "0.8.0" +skylib_version = "0.9.0" http_archive( name = "bazel_skylib", type = "tar.gz", - url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version), - sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", + url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version), + sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0", ) load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "1.0.0", - maximum_bazel_version = "1.2.1") +versions.check(minimum_bazel_version = "2.0.0") # ABSL cpp library lts_2020_02_25 @@ -93,11 +92,13 @@ http_archive( build_file = "@//third_party:libyuv.BUILD", ) +# Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download. +# ...but the Java download is currently broken, so we use the "source" download. http_archive( name = "com_google_protobuf_javalite", - sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc", - strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516", - urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"], + sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9", + strip_prefix = "protobuf-3.11.4", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"], ) http_archive( @@ -117,23 +118,18 @@ http_archive( ], ) -# 2020-02-12 -# The last commit before TensorFlow switched to Bazel 2.0 -_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328" -_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13" +# 2020-04-01 +_TENSORFLOW_GIT_COMMIT = "805e47cea96c7e8c6fccf494d40a2392dc99fdd8" +_TENSORFLOW_SHA256= "9ee3ae604c2e1345ac60345becee6d659364721513f9cb8652eb2e7138320ca5" http_archive( name = "org_tensorflow", urls = [ "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, ], - # A compatibility patch patches = [ - "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff", - # Updates for XNNPACK: https://github.com/tensorflow/tensorflow/commit/cfc31e324c8de6b52f752a39cb161d99d853ca99 - "@//third_party:org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff", - # CpuInfo's build rule fixes. - "@//third_party:org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff", + "@//third_party:org_tensorflow_compatibility_fixes.diff", + "@//third_party:org_tensorflow_protobuf_updates.diff", ], patch_args = [ "-p1", @@ -158,10 +154,6 @@ http_archive( sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1" ) -# Please run -# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \ -# libopencv-calib3d-dev libopencv-features2d-dev \ -# libopencv-imgproc-dev libopencv-video-dev new_local_repository( name = "linux_opencv", build_file = "@//third_party:opencv_linux.BUILD", @@ -174,7 +166,6 @@ new_local_repository( path = "/usr" ) -# Please run $ brew install opencv@3 new_local_repository( name = "macos_opencv", build_file = "@//third_party:opencv_macos.BUILD", @@ -207,79 +198,6 @@ http_archive( url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip", ) -RULES_JVM_EXTERNAL_TAG = "2.2" -RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6" - -http_archive( - name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - sha256 = RULES_JVM_EXTERNAL_SHA, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = [ - "androidx.annotation:annotation:aar:1.1.0", - "androidx.appcompat:appcompat:aar:1.1.0-rc01", - "androidx.camera:camera-core:aar:1.0.0-alpha06", - "androidx.camera:camera-camera2:aar:1.0.0-alpha06", - "androidx.constraintlayout:constraintlayout:aar:1.1.3", - "androidx.core:core:aar:1.1.0-rc03", - "androidx.legacy:legacy-support-v4:aar:1.0.0", - "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", - "com.google.android.material:material:aar:1.0.0-rc01", - ], - repositories = [ - "https://dl.google.com/dl/android/maven2", - "https://repo1.maven.org/maven2", - ], -) - -maven_server( - name = "google_server", - url = "https://dl.google.com/dl/android/maven2", -) - -maven_jar( - name = "androidx_lifecycle", - artifact = "androidx.lifecycle:lifecycle-common:2.0.0", - sha1 = "e070ffae07452331bc5684734fce6831d531785c", - server = "google_server", -) - -maven_jar( - name = "androidx_concurrent_futures", - artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03", - sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae", - server = "google_server", -) - -maven_jar( - name = "com_google_guava_android", - artifact = "com.google.guava:guava:27.0.1-android", - sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d", -) - -maven_jar( - name = "com_google_common_flogger", - artifact = "com.google.flogger:flogger:0.3.1", - sha1 = "585030fe1ec709760cbef997a459729fb965df0e", -) - -maven_jar( - name = "com_google_common_flogger_system_backend", - artifact = "com.google.flogger:flogger-system-backend:0.3.1", - sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9", -) - -maven_jar( - name = "com_google_code_findbugs", - artifact = "com.google.code.findbugs:jsr305:3.0.2", - sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", -) - # You may run setup_android.sh to install Android SDK and NDK. android_ndk_repository( name = "androidndk", @@ -293,9 +211,15 @@ android_sdk_repository( http_archive( name = "build_bazel_rules_apple", - sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", - strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", - url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", + sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42", + url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz", + patches = [ + # Bypass checking ios unit test runner when building MP ios applications. + "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff" + ], + patch_args = [ + "-p1", + ], ) load( @@ -329,3 +253,49 @@ http_archive( build_file = "@//third_party:google_toolbox_for_mac.BUILD", ) +# Maven dependencies. + +RULES_JVM_EXTERNAL_TAG = "3.2" +RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af" + +http_archive( + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + sha256 = RULES_JVM_EXTERNAL_SHA, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:defs.bzl", "maven_install") + +# Important: there can only be one maven_install rule. Add new maven deps here. +maven_install( + name = "maven", + artifacts = [ + "junit:junit:4.12", + "androidx.test.espresso:espresso-core:3.1.1", + "org.hamcrest:hamcrest-library:1.3", + "androidx.concurrent:concurrent-futures:1.0.0-alpha03", + "androidx.lifecycle:lifecycle-common:2.0.0", + "androidx.annotation:annotation:aar:1.1.0", + "androidx.appcompat:appcompat:aar:1.1.0-rc01", + "androidx.camera:camera-core:aar:1.0.0-alpha06", + "androidx.camera:camera-camera2:aar:1.0.0-alpha06", + "androidx.constraintlayout:constraintlayout:aar:1.1.3", + "androidx.core:core:aar:1.1.0-rc03", + "androidx.legacy:legacy-support-v4:aar:1.0.0", + "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", + "com.google.android.material:material:aar:1.0.0-rc01", + "com.google.code.findbugs:jsr305:3.0.2", + "com.google.flogger:flogger-system-backend:0.3.1", + "com.google.flogger:flogger:0.3.1", + "com.google.guava:guava:27.0.1-android", + ], + repositories = [ + "https://jcenter.bintray.com", + "https://maven.google.com", + "https://dl.google.com/dl/android/maven2", + "https://repo1.maven.org/maven2", + ], + fetch_sources = True, + version_conflict_policy = "pinned", +) diff --git a/mediapipe/BUILD b/mediapipe/BUILD index d8b37ef20..82bb3c83e 100644 --- a/mediapipe/BUILD +++ b/mediapipe/BUILD @@ -14,6 +14,9 @@ licenses(["notice"]) # Apache 2.0 +# Note: yes, these need to use "//external:android/crosstool", not +# @androidndk//:default_crosstool. + config_setting( name = "android", values = {"crosstool_top": "//external:android/crosstool"}, diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 65c426489..1c9910fc7 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -230,6 +230,7 @@ cc_library( "//mediapipe/framework:packet", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:ret_check", @@ -257,6 +258,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/util:render_data_cc_proto", + "@org_tensorflow//tensorflow/lite:framework", ], alwayslink = 1, ) @@ -779,6 +781,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/formats:matrix", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/util:resource_util", diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc index bd4e554e1..1d0f7824d 100644 --- a/mediapipe/calculators/core/begin_loop_calculator.cc +++ b/mediapipe/calculators/core/begin_loop_calculator.cc @@ -18,6 +18,7 @@ #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/rect.pb.h" namespace mediapipe { @@ -37,4 +38,8 @@ typedef BeginLoopCalculator> BeginLoopDetectionCalculator; REGISTER_CALCULATOR(BeginLoopDetectionCalculator); +// A calculator to process std::vector. +typedef BeginLoopCalculator> BeginLoopMatrixCalculator; +REGISTER_CALCULATOR(BeginLoopMatrixCalculator); + } // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc index 61e8c6ac0..fb02f7618 100644 --- a/mediapipe/calculators/core/end_loop_calculator.cc +++ b/mediapipe/calculators/core/end_loop_calculator.cc @@ -20,6 +20,7 @@ #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/util/render_data.pb.h" +#include "tensorflow/lite/interpreter.h" namespace mediapipe { @@ -42,4 +43,7 @@ typedef EndLoopCalculator> EndLoopClassificationListCalculator; REGISTER_CALCULATOR(EndLoopClassificationListCalculator); +typedef EndLoopCalculator> EndLoopTensorCalculator; +REGISTER_CALCULATOR(EndLoopTensorCalculator); + } // namespace mediapipe diff --git a/mediapipe/calculators/core/split_vector_calculator.cc b/mediapipe/calculators/core/split_vector_calculator.cc index d993387df..100507c99 100644 --- a/mediapipe/calculators/core/split_vector_calculator.cc +++ b/mediapipe/calculators/core/split_vector_calculator.cc @@ -18,6 +18,7 @@ #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/rect.pb.h" #include "tensorflow/lite/interpreter.h" @@ -57,6 +58,9 @@ typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false> SplitNormalizedRectVectorCalculator; REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator); +typedef SplitVectorCalculator SplitMatrixVectorCalculator; +REGISTER_CALCULATOR(SplitMatrixVectorCalculator); + #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true> MovableSplitGlBufferVectorCalculator; diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index 1ef87d314..8df493d22 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -86,6 +86,14 @@ mediapipe_cc_proto_library( deps = [":opencv_image_encoder_calculator_proto"], ) +mediapipe_cc_proto_library( + name = "opencv_encoded_image_to_image_frame_calculator_cc_proto", + srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"], + cc_deps = ["//mediapipe/framework:calculator_cc_proto"], + visibility = ["//visibility:public"], + deps = [":opencv_encoded_image_to_image_frame_calculator_proto"], +) + mediapipe_cc_proto_library( name = "mask_overlay_calculator_cc_proto", srcs = ["mask_overlay_calculator.proto"], @@ -172,6 +180,7 @@ cc_library( srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"], visibility = ["//visibility:public"], deps = [ + ":opencv_encoded_image_to_image_frame_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/port:opencv_imgcodecs", @@ -557,6 +566,27 @@ proto_library( deps = ["//mediapipe/framework:calculator_proto"], ) +proto_library( + name = "opencv_encoded_image_to_image_frame_calculator_proto", + srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"], + visibility = ["//visibility:public"], + deps = ["//mediapipe/framework:calculator_proto"], +) + +proto_library( + name = "feature_detector_calculator_proto", + srcs = ["feature_detector_calculator.proto"], + deps = ["//mediapipe/framework:calculator_proto"], +) + +mediapipe_cc_proto_library( + name = "feature_detector_calculator_cc_proto", + srcs = ["feature_detector_calculator.proto"], + cc_deps = ["//mediapipe/framework:calculator_cc_proto"], + visibility = ["//visibility:public"], + deps = [":feature_detector_calculator_proto"], +) + cc_library( name = "mask_overlay_calculator", srcs = ["mask_overlay_calculator.cc"], @@ -572,3 +602,30 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "feature_detector_calculator", + srcs = ["feature_detector_calculator.cc"], + visibility = ["//mediapipe:__subpackages__"], + deps = [ + ":feature_detector_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:video_stream_header", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_features2d", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:threadpool", + "//mediapipe/framework/tool:options_util", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/synchronization", + "@org_tensorflow//tensorflow/lite:framework", + ], + alwayslink = 1, +) diff --git a/mediapipe/calculators/image/feature_detector_calculator.cc b/mediapipe/calculators/image/feature_detector_calculator.cc new file mode 100644 index 000000000..9f873740e --- /dev/null +++ b/mediapipe/calculators/image/feature_detector_calculator.cc @@ -0,0 +1,210 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "absl/memory/memory.h" +#include "absl/synchronization/blocking_counter.h" +#include "mediapipe/calculators/image/feature_detector_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/video_stream_header.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_features2d_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/threadpool.h" +#include "mediapipe/framework/tool/options_util.h" +#include "tensorflow/lite/interpreter.h" + +namespace mediapipe { + +const char kOptionsTag[] = "OPTIONS"; +const int kPatchSize = 32; +const int kNumThreads = 16; + +// A calculator to apply local feature detection. +// Input stream: +// IMAGE: Input image frame of type ImageFrame from video stream. +// Output streams: +// FEATURES: The detected keypoints from input image as vector. +// PATCHES: Optional output the extracted patches as vector +class FeatureDetectorCalculator : public CalculatorBase { + public: + ~FeatureDetectorCalculator() override = default; + + static ::mediapipe::Status GetContract(CalculatorContract* cc); + + ::mediapipe::Status Open(CalculatorContext* cc) override; + ::mediapipe::Status Process(CalculatorContext* cc) override; + + private: + FeatureDetectorCalculatorOptions options_; + cv::Ptr feature_detector_; + std::unique_ptr<::mediapipe::ThreadPool> pool_; + + // Create image pyramid based on input image. + void ComputeImagePyramid(const cv::Mat& input_image, + std::vector* image_pyramid); + + // Extract the patch for single feature with image pyramid. + cv::Mat ExtractPatch(const cv::KeyPoint& feature, + const std::vector& image_pyramid); +}; + +REGISTER_CALCULATOR(FeatureDetectorCalculator); + +::mediapipe::Status FeatureDetectorCalculator::GetContract( + CalculatorContract* cc) { + if (cc->Inputs().HasTag("IMAGE")) { + cc->Inputs().Tag("IMAGE").Set(); + } + if (cc->Outputs().HasTag("FEATURES")) { + cc->Outputs().Tag("FEATURES").Set>(); + } + if (cc->Outputs().HasTag("LANDMARKS")) { + cc->Outputs().Tag("LANDMARKS").Set(); + } + if (cc->Outputs().HasTag("PATCHES")) { + cc->Outputs().Tag("PATCHES").Set>(); + } + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) { + options_ = + tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag) + .GetExtension(FeatureDetectorCalculatorOptions::ext); + feature_detector_ = cv::ORB::create( + options_.max_features(), options_.scale_factor(), + options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE); + pool_ = absl::make_unique<::mediapipe::ThreadPool>("ThreadPool", kNumThreads); + pool_->StartWorkers(); + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) { + const Timestamp& timestamp = cc->InputTimestamp(); + if (timestamp == Timestamp::PreStream()) { + // Indicator packet. + return ::mediapipe::OkStatus(); + } + InputStream* input_frame = &(cc->Inputs().Tag("IMAGE")); + cv::Mat input_view = formats::MatView(&input_frame->Get()); + cv::Mat grayscale_view; + cv::cvtColor(input_view, grayscale_view, cv::COLOR_RGB2GRAY); + + std::vector keypoints; + feature_detector_->detect(grayscale_view, keypoints); + if (keypoints.size() > options_.max_features()) { + keypoints.resize(options_.max_features()); + } + + if (cc->Outputs().HasTag("FEATURES")) { + auto features_ptr = absl::make_unique>(keypoints); + cc->Outputs().Tag("FEATURES").Add(features_ptr.release(), timestamp); + } + + if (cc->Outputs().HasTag("LANDMARKS")) { + auto landmarks_ptr = absl::make_unique(); + for (int j = 0; j < keypoints.size(); ++j) { + auto feature_landmark = landmarks_ptr->add_landmark(); + feature_landmark->set_x(keypoints[j].pt.x / grayscale_view.cols); + feature_landmark->set_y(keypoints[j].pt.y / grayscale_view.rows); + } + cc->Outputs().Tag("LANDMARKS").Add(landmarks_ptr.release(), timestamp); + } + + if (cc->Outputs().HasTag("PATCHES")) { + std::vector image_pyramid; + ComputeImagePyramid(grayscale_view, &image_pyramid); + std::vector patch_mat; + patch_mat.resize(keypoints.size()); + absl::BlockingCounter counter(keypoints.size()); + for (int i = 0; i < keypoints.size(); i++) { + pool_->Schedule( + [this, &image_pyramid, &keypoints, &patch_mat, i, &counter] { + patch_mat[i] = ExtractPatch(keypoints[i], image_pyramid); + counter.DecrementCount(); + }); + } + counter.Wait(); + const int batch_size = options_.max_features(); + auto patches = absl::make_unique>(); + TfLiteTensor tensor; + tensor.type = kTfLiteFloat32; + tensor.dims = TfLiteIntArrayCreate(4); + tensor.dims->data[0] = batch_size; + tensor.dims->data[1] = kPatchSize; + tensor.dims->data[2] = kPatchSize; + tensor.dims->data[3] = 1; + int num_bytes = batch_size * kPatchSize * kPatchSize * sizeof(float); + tensor.data.data = malloc(num_bytes); + tensor.bytes = num_bytes; + tensor.allocation_type = kTfLiteArenaRw; + float* tensor_buffer = tensor.data.f; + for (int i = 0; i < keypoints.size(); i++) { + for (int j = 0; j < patch_mat[i].rows; ++j) { + for (int k = 0; k < patch_mat[i].cols; ++k) { + *tensor_buffer++ = patch_mat[i].at(j, k) / 128.0f - 1.0f; + } + } + } + for (int i = keypoints.size() * kPatchSize * kPatchSize; i < num_bytes / 4; + i++) { + *tensor_buffer++ = 0; + } + + patches->emplace_back(tensor); + cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp); + } + + return ::mediapipe::OkStatus(); +} + +void FeatureDetectorCalculator::ComputeImagePyramid( + const cv::Mat& input_image, std::vector* image_pyramid) { + cv::Mat tmp_image = input_image; + cv::Mat src_image = input_image; + for (int i = 0; i < options_.pyramid_level(); ++i) { + image_pyramid->push_back(src_image); + cv::resize(src_image, tmp_image, cv::Size(), 1.0f / options_.scale_factor(), + 1.0f / options_.scale_factor()); + src_image = tmp_image; + } +} + +cv::Mat FeatureDetectorCalculator::ExtractPatch( + const cv::KeyPoint& feature, const std::vector& image_pyramid) { + cv::Mat img = image_pyramid[feature.octave]; + float scale_factor = 1 / pow(options_.scale_factor(), feature.octave); + cv::Point2f center = + cv::Point2f(feature.pt.x * scale_factor, feature.pt.y * scale_factor); + cv::Mat rot = cv::getRotationMatrix2D(center, feature.angle, 1.0); + rot.at(0, 2) += kPatchSize / 2 - center.x; + rot.at(1, 2) += kPatchSize / 2 - center.y; + cv::Mat cropped_img; + // perform the affine transformation + cv::warpAffine(img, cropped_img, rot, cv::Size(kPatchSize, kPatchSize), + cv::INTER_LINEAR); + return cropped_img; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/image/feature_detector_calculator.proto b/mediapipe/calculators/image/feature_detector_calculator.proto new file mode 100644 index 000000000..7bff23572 --- /dev/null +++ b/mediapipe/calculators/image/feature_detector_calculator.proto @@ -0,0 +1,24 @@ +// Options for FeatureDetectorCalculator +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message FeatureDetectorCalculatorOptions { + extend CalculatorOptions { + optional FeatureDetectorCalculatorOptions ext = 278741680; + } + + // Set to true if output patches, otherwise only output cv::KeyPoint + optional bool output_patch = 1; + + // The max number of detected features. + optional int32 max_features = 2 [default = 200]; + + // The number of pyramid levels. + optional int32 pyramid_level = 3 [default = 4]; + + // Pyramid decimation ratio. + optional float scale_factor = 4 [default = 1.2]; +} diff --git a/mediapipe/calculators/image/image_cropping_calculator.cc b/mediapipe/calculators/image/image_cropping_calculator.cc index 79a3221ea..9500941cb 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.cc +++ b/mediapipe/calculators/image/image_cropping_calculator.cc @@ -219,8 +219,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator); const auto& input_img = cc->Inputs().Tag(kImageTag).Get(); cv::Mat input_mat = formats::MatView(&input_img); - auto [target_width, target_height, rect_center_x, rect_center_y, rotation] = - GetCropSpecs(cc, input_img.Width(), input_img.Height()); + RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height()); + int target_width = specs.width, target_height = specs.height, + rect_center_x = specs.center_x, rect_center_y = specs.center_y; + float rotation = specs.rotation; // Get border mode and value for OpenCV. int border_mode; @@ -403,8 +405,10 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height, int* dst_width, int* dst_height) { - auto [crop_width, crop_height, x_center, y_center, rotation] = - GetCropSpecs(cc, src_width, src_height); + RectSpec specs = GetCropSpecs(cc, src_width, src_height); + int crop_width = specs.width, crop_height = specs.height, + x_center = specs.center_x, y_center = specs.center_y; + float rotation = specs.rotation; const float half_width = crop_width / 2.0f; const float half_height = crop_height / 2.0f; diff --git a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc index 445329009..ae87d7511 100644 --- a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc +++ b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" @@ -34,7 +35,11 @@ namespace mediapipe { class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc); + ::mediapipe::Status Open(CalculatorContext* cc) override; ::mediapipe::Status Process(CalculatorContext* cc) override; + + private: + mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_; }; ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract( @@ -44,13 +49,29 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase { return ::mediapipe::OkStatus(); } +::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open( + CalculatorContext* cc) { + options_ = + cc->Options(); + return ::mediapipe::OkStatus(); +} + ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process( CalculatorContext* cc) { const std::string& contents = cc->Inputs().Index(0).Get(); const std::vector contents_vector(contents.begin(), contents.end()); - cv::Mat decoded_mat = - cv::imdecode(contents_vector, -1 /* return the loaded image as-is */); - + cv::Mat decoded_mat; + if (options_.apply_orientation_from_exif_data()) { + // We want to respect the orientation from the EXIF data, which + // IMREAD_UNCHANGED ignores, but otherwise we want to be as permissive as + // possible with our reading flags. Therefore, we use IMREAD_ANYCOLOR and + // IMREAD_ANYDEPTH. + decoded_mat = cv::imdecode(contents_vector, + cv::IMREAD_ANYCOLOR | cv::IMREAD_ANYDEPTH); + } else { + // Return the loaded image as-is + decoded_mat = cv::imdecode(contents_vector, cv::IMREAD_UNCHANGED); + } ImageFormat::Format image_format = ImageFormat::UNKNOWN; cv::Mat output_mat; switch (decoded_mat.channels()) { @@ -70,7 +91,8 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase { << "Unsupported number of channels: " << decoded_mat.channels(); } std::unique_ptr output_frame = absl::make_unique( - image_format, decoded_mat.size().width, decoded_mat.size().height); + image_format, decoded_mat.size().width, decoded_mat.size().height, + ImageFrame::kGlDefaultAlignmentBoundary); output_mat.copyTo(formats::MatView(output_frame.get())); cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp()); return ::mediapipe::OkStatus(); diff --git a/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto new file mode 100644 index 000000000..3ec29bddf --- /dev/null +++ b/mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto @@ -0,0 +1,30 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message OpenCvEncodedImageToImageFrameCalculatorOptions { + extend CalculatorOptions { + optional OpenCvEncodedImageToImageFrameCalculatorOptions ext = 303447308; + } + + // If set, we will attempt to automatically apply the orientation specified by + // the image's EXIF data when loading the image. Otherwise, the image data + // will be loaded as-is. + optional bool apply_orientation_from_exif_data = 1 [default = false]; +} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc index 03ce08f7b..6100ddceb 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc @@ -51,10 +51,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH"; #endif } -// If options.convert_signature_to_tags() will convert letters to uppercase -// and replace /'s with _'s. If set, this enables the standard SavedModel -// classification, regression, and prediction signatures to be used as -// uppercase INPUTS and OUTPUTS tags for streams. +// If options.convert_signature_to_tags() is set, will convert letters to +// uppercase and replace /'s and -'s with _'s. This enables the standard +// SavedModel classification, regression, and prediction signatures to be used +// as uppercase INPUTS and OUTPUTS tags for streams and supports other common +// patterns. const std::string MaybeConvertSignatureToTag( const std::string& name, const TensorFlowSessionFromSavedModelCalculatorOptions& options) { @@ -64,6 +65,7 @@ const std::string MaybeConvertSignatureToTag( std::transform(name.begin(), name.end(), output.begin(), [](unsigned char c) { return std::toupper(c); }); output = absl::StrReplaceAll(output, {{"/", "_"}}); + output = absl::StrReplaceAll(output, {{"-", "_"}}); return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto index da5edb91c..66e03d893 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto @@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelCalculatorOptions { // The name of the generic signature to load into the mapping from tags to // tensor names. optional string signature_name = 2 [default = "serving_default"]; - // Whether to convert the signature keys to uppercase and switch /'s to - // _'s, which enables standard signatures to be used as Tags. + // Whether to convert the signature keys to uppercase as well as switch /'s + // and -'s to _'s, which enables common signatures to be used as Tags. optional bool convert_signature_to_tags = 3 [default = true]; // If true, saved_model_path can have multiple exported models in // subdirectories saved_model_path/%08d and the alphabetically last (i.e., diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc index a2aaa1e9a..4e956e759 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc @@ -53,10 +53,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH"; #endif } -// If options.convert_signature_to_tags() will convert letters to uppercase -// and replace /'s with _'s. If set, this enables the standard SavedModel -// classification, regression, and prediction signatures to be used as -// uppercase INPUTS and OUTPUTS tags for streams. +// If options.convert_signature_to_tags() is set, will convert letters to +// uppercase and replace /'s and -'s with _'s. This enables the standard +// SavedModel classification, regression, and prediction signatures to be used +// as uppercase INPUTS and OUTPUTS tags for streams and supports other common +// patterns. const std::string MaybeConvertSignatureToTag( const std::string& name, const TensorFlowSessionFromSavedModelGeneratorOptions& options) { @@ -66,6 +67,7 @@ const std::string MaybeConvertSignatureToTag( std::transform(name.begin(), name.end(), output.begin(), [](unsigned char c) { return std::toupper(c); }); output = absl::StrReplaceAll(output, {{"/", "_"}}); + output = absl::StrReplaceAll(output, {{"-", "_"}}); return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto index 7e33f7518..2dab09242 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto @@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelGeneratorOptions { // The name of the generic signature to load into the mapping from tags to // tensor names. optional string signature_name = 2 [default = "serving_default"]; - // Whether to convert the signature keys to uppercase and switch /'s to - // _'s, which enables standard signatures to be used as Tags. + // Whether to convert the signature keys to uppercase as well as switch /'s + // and -'s to _'s, which enables common signatures to be used as Tags. optional bool convert_signature_to_tags = 3 [default = true]; // If true, saved_model_path can have multiple exported models in // subdirectories saved_model_path/%08d and the alphabetically last (i.e., diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index 45ef317c8..abc83f68e 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -451,7 +451,7 @@ cc_library( "//mediapipe:android": [ "//mediapipe/util/android/file/base", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "//mediapipe/util/android/file/base", ], "//mediapipe:macos": [ diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index 665bd89f7..954cfea9f 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -673,7 +673,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); const auto& input_indices = interpreter_->inputs(); gpu_data_in_.resize(input_indices.size()); for (int i = 0; i < input_indices.size(); ++i) { - const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]); + const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]); gpu_data_in_[i] = absl::make_unique(); gpu_data_in_[i]->elements = 1; for (int d = 0; d < tensor->dims->size; ++d) { diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc index bc4199969..52f9b85b7 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc @@ -145,7 +145,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get() : options_.flip_horizontally(); - flip_horizontally_ = + flip_vertically_ = cc->InputSidePackets().HasTag("FLIP_VERTICALLY") ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get() : options_.flip_vertically(); diff --git a/mediapipe/calculators/tflite/util.h b/mediapipe/calculators/tflite/util.h index 53e0927af..3a60441cb 100644 --- a/mediapipe/calculators/tflite/util.h +++ b/mediapipe/calculators/tflite/util.h @@ -15,11 +15,11 @@ #ifndef MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_ #define MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_ -#define RET_CHECK_CALL(call) \ - do { \ - const auto status = (call); \ - if (ABSL_PREDICT_FALSE(!status.ok())) \ - return ::mediapipe::InternalError(status.error_message()); \ +#define RET_CHECK_CALL(call) \ + do { \ + const auto status = (call); \ + if (ABSL_PREDICT_FALSE(!status.ok())) \ + return ::mediapipe::InternalError(status.message()); \ } while (0); #endif // MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_ diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 3d2210ca4..2aae53d8e 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -321,7 +321,7 @@ cc_library( "//mediapipe:android": [ "//mediapipe/util/android/file/base", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "//mediapipe/util/android/file/base", ], "//mediapipe:macos": [ @@ -349,7 +349,7 @@ cc_library( "//mediapipe:android": [ "//mediapipe/util/android/file/base", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "//mediapipe/util/android/file/base", ], "//mediapipe:macos": [ @@ -926,7 +926,7 @@ cc_library( "//mediapipe:android": [ "//mediapipe/util/android/file/base", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "//mediapipe/util/android/file/base", ], "//mediapipe:macos": [ @@ -971,9 +971,9 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "//mediapipe/util:resource_util", ], alwayslink = 1, ) diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index ead18a76d..29cf74962 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -55,7 +55,7 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT // When using GPU, this color will become transparent when the calculator // merges the annotation overlay with the image frame. As a result, drawing in // this color is not supported and it should be set to something unlikely used. -constexpr int kAnnotationBackgroundColor[] = {100, 101, 102}; +constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value. } // namespace // A calculator for rendering data on images. @@ -491,11 +491,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator); if (format != mediapipe::ImageFormat::SRGBA && format != mediapipe::ImageFormat::SRGB) RET_CHECK_FAIL() << "Unsupported GPU input format: " << format; - - image_mat = absl::make_unique( - height_, width_, CV_8UC3, - cv::Scalar(kAnnotationBackgroundColor[0], kAnnotationBackgroundColor[1], - kAnnotationBackgroundColor[2])); + image_mat = absl::make_unique(height_, width_, CV_8UC3); + memset(image_mat->data, kAnnotationBackgroundColor, + height_ * width_ * image_mat->elemSize()); } else { image_mat = absl::make_unique( options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3, @@ -617,9 +615,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator); glUniform1i(glGetUniformLocation(program_, "input_frame"), 1); glUniform1i(glGetUniformLocation(program_, "overlay"), 2); glUniform3f(glGetUniformLocation(program_, "transparent_color"), - kAnnotationBackgroundColor[0] / 255.0, - kAnnotationBackgroundColor[1] / 255.0, - kAnnotationBackgroundColor[2] / 255.0); + kAnnotationBackgroundColor / 255.0, + kAnnotationBackgroundColor / 255.0, + kAnnotationBackgroundColor / 255.0); // Init texture for opencv rendered frame. const auto& input_frame = diff --git a/mediapipe/calculators/util/labels_to_render_data_calculator.cc b/mediapipe/calculators/util/labels_to_render_data_calculator.cc index 722f4480c..fafedba5b 100644 --- a/mediapipe/calculators/util/labels_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/labels_to_render_data_calculator.cc @@ -128,16 +128,19 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator); } else { const std::vector& label_vector = cc->Inputs().Tag("LABELS").Get>(); - std::vector score_vector; - if (cc->Inputs().HasTag("SCORES")) { - score_vector = cc->Inputs().Tag("SCORES").Get>(); - } - CHECK_EQ(label_vector.size(), score_vector.size()); labels.resize(label_vector.size()); - scores.resize(label_vector.size()); for (int i = 0; i < label_vector.size(); ++i) { labels[i] = label_vector[i]; - scores[i] = score_vector[i]; + } + + if (cc->Inputs().HasTag("SCORES")) { + std::vector score_vector = + cc->Inputs().Tag("SCORES").Get>(); + CHECK_EQ(label_vector.size(), score_vector.size()); + scores.resize(label_vector.size()); + for (int i = 0; i < label_vector.size(); ++i) { + scores[i] = score_vector[i]; + } } } diff --git a/mediapipe/calculators/util/local_file_contents_calculator.cc b/mediapipe/calculators/util/local_file_contents_calculator.cc index 9f8d17724..254a552c9 100644 --- a/mediapipe/calculators/util/local_file_contents_calculator.cc +++ b/mediapipe/calculators/util/local_file_contents_calculator.cc @@ -16,34 +16,80 @@ #include #include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/status.h" +#include "mediapipe/util/resource_util.h" namespace mediapipe { + +namespace { + +constexpr char kFilePathTag[] = "FILE_PATH"; +constexpr char kContentsTag[] = "CONTENTS"; + +} // namespace + // The calculator takes the path to the local file as an input side packet and // outputs the contents of that file. // +// NOTE: file loading can be batched by providing multiple input/output side +// packets. +// // Example config: // node { // calculator: "LocalFileContentsCalculator" // input_side_packet: "FILE_PATH:file_path" // output_side_packet: "CONTENTS:contents" // } +// +// node { +// calculator: "LocalFileContentsCalculator" +// input_side_packet: "FILE_PATH:0:file_path1" +// input_side_packet: "FILE_PATH:1:file_path2" +// ... +// output_side_packet: "CONTENTS:0:contents1" +// output_side_packet: "CONTENTS:1:contents2" +// ... +// } class LocalFileContentsCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Tag("FILE_PATH").Set(); - cc->OutputSidePackets().Tag("CONTENTS").Set(); + RET_CHECK(cc->InputSidePackets().HasTag(kFilePathTag)) + << "Missing PATH input side packet(s)"; + RET_CHECK(cc->OutputSidePackets().HasTag(kContentsTag)) + << "Missing CONTENTS output side packet(s)"; + + RET_CHECK_EQ(cc->InputSidePackets().NumEntries(kFilePathTag), + cc->OutputSidePackets().NumEntries(kContentsTag)) + << "Same number of input streams and output streams is required."; + + for (CollectionItemId id = cc->InputSidePackets().BeginId(kFilePathTag); + id != cc->InputSidePackets().EndId(kFilePathTag); ++id) { + cc->InputSidePackets().Get(id).Set(); + } + + for (CollectionItemId id = cc->OutputSidePackets().BeginId(kContentsTag); + id != cc->OutputSidePackets().EndId(kContentsTag); ++id) { + cc->OutputSidePackets().Get(id).Set(); + } + return ::mediapipe::OkStatus(); } ::mediapipe::Status Open(CalculatorContext* cc) override { - std::string contents; - MP_RETURN_IF_ERROR(mediapipe::file::GetContents( - cc->InputSidePackets().Tag("FILE_PATH").Get(), &contents)); - cc->OutputSidePackets() - .Tag("CONTENTS") - .Set(MakePacket(std::move(contents))); + CollectionItemId input_id = cc->InputSidePackets().BeginId(kFilePathTag); + CollectionItemId output_id = cc->OutputSidePackets().BeginId(kContentsTag); + // Number of inputs and outpus is the same according to the contract. + for (; input_id != cc->InputSidePackets().EndId(kFilePathTag); + ++input_id, ++output_id) { + std::string file_path = + cc->InputSidePackets().Get(input_id).Get(); + ASSIGN_OR_RETURN(file_path, PathToResourceAsFile(file_path)); + + std::string contents; + MP_RETURN_IF_ERROR(GetResourceContents(file_path, &contents)); + cc->OutputSidePackets().Get(output_id).Set( + MakePacket(std::move(contents))); + } return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc b/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc index d76498d87..3979d14fc 100644 --- a/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -76,14 +78,16 @@ void AddTimedBoxProtoToRenderData( RenderAnnotation::Text* text = label_annotation->mutable_text(); text->set_display_text(box_proto.label()); text->set_normalized(true); - constexpr float text_left_start = 0.3f; + constexpr float text_left_start = 0.2f; text->set_left((1.0f - text_left_start) * box_proto.left() + text_left_start * box_proto.right()); constexpr float text_baseline = 0.6f; text->set_baseline(text_baseline * box_proto.bottom() + (1.0f - text_baseline) * box_proto.top()); - constexpr float text_height = 0.2f; - text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height); + constexpr float text_height = 0.1f; + text->set_font_height(std::min(box_proto.bottom() - box_proto.top(), + box_proto.right() - box_proto.left()) * + text_height); } } diff --git a/mediapipe/calculators/video/BUILD b/mediapipe/calculators/video/BUILD index f4fca811a..d171f8af1 100644 --- a/mediapipe/calculators/video/BUILD +++ b/mediapipe/calculators/video/BUILD @@ -65,6 +65,26 @@ proto_library( ], ) +proto_library( + name = "tracked_detection_manager_calculator_proto", + srcs = ["tracked_detection_manager_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_proto", + "//mediapipe/util/tracking:tracked_detection_manager_config_proto", + ], +) + +proto_library( + name = "box_detector_calculator_proto", + srcs = ["box_detector_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_proto", + "//mediapipe/util/tracking:box_detector_proto", + ], +) + proto_library( name = "video_pre_stream_calculator_proto", srcs = ["video_pre_stream_calculator.proto"], @@ -107,6 +127,28 @@ mediapipe_cc_proto_library( deps = [":box_tracker_calculator_proto"], ) +mediapipe_cc_proto_library( + name = "tracked_detection_manager_calculator_cc_proto", + srcs = ["tracked_detection_manager_calculator.proto"], + cc_deps = [ + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/util/tracking:tracked_detection_manager_config_cc_proto", + ], + visibility = ["//visibility:public"], + deps = [":tracked_detection_manager_calculator_proto"], +) + +mediapipe_cc_proto_library( + name = "box_detector_calculator_cc_proto", + srcs = ["box_detector_calculator.proto"], + cc_deps = [ + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/util/tracking:box_detector_cc_proto", + ], + visibility = ["//visibility:public"], + deps = [":box_detector_calculator_proto"], +) + mediapipe_cc_proto_library( name = "video_pre_stream_calculator_cc_proto", srcs = ["video_pre_stream_calculator.proto"], @@ -279,11 +321,54 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "box_detector_calculator", + srcs = ["box_detector_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":box_detector_calculator_cc_proto", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:video_stream_header", # fixdeps: keep -- required for exobazel build. + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_features2d", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/util:resource_util", + "//mediapipe/util/tracking", + "//mediapipe/util/tracking:box_detector", + "//mediapipe/util/tracking:box_tracker", + "//mediapipe/util/tracking:box_tracker_cc_proto", + "//mediapipe/util/tracking:flow_packager_cc_proto", + "//mediapipe/util/tracking:tracking_visualization_utilities", + ] + select({ + "//mediapipe:android": [ + "//mediapipe/util/android/file/base", + ], + "//mediapipe:ios": [ + "//mediapipe/util/android/file/base", + ], + "//mediapipe:macos": [ + "//mediapipe/framework/port:file_helpers", + ], + "//conditions:default": [ + "//mediapipe/framework/port:file_helpers", + ], + }), + alwayslink = 1, +) + cc_library( name = "tracked_detection_manager_calculator", srcs = ["tracked_detection_manager_calculator.cc"], visibility = ["//visibility:public"], deps = [ + ":tracked_detection_manager_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:location_data_cc_proto", diff --git a/mediapipe/calculators/video/box_detector_calculator.cc b/mediapipe/calculators/video/box_detector_calculator.cc new file mode 100644 index 000000000..d9afdd333 --- /dev/null +++ b/mediapipe/calculators/video/box_detector_calculator.cc @@ -0,0 +1,393 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "absl/memory/memory.h" +#include "absl/strings/numbers.h" +#include "mediapipe/calculators/video/box_detector_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/video_stream_header.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_features2d_inc.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/util/resource_util.h" +#include "mediapipe/util/tracking/box_detector.h" +#include "mediapipe/util/tracking/box_tracker.h" +#include "mediapipe/util/tracking/box_tracker.pb.h" +#include "mediapipe/util/tracking/flow_packager.pb.h" +#include "mediapipe/util/tracking/tracking.h" +#include "mediapipe/util/tracking/tracking_visualization_utilities.h" + +#if defined(MEDIAPIPE_MOBILE) +#include "mediapipe/util/android/file/base/file.h" +#include "mediapipe/util/android/file/base/helpers.h" +#else +#include "mediapipe/framework/port/file_helpers.h" +#endif + +namespace mediapipe { + +// A calculator to detect reappeared box positions from single frame. +// +// Input stream: +// TRACKING: Input tracking data (proto TrackingData) containing features and +// descriptors. +// VIDEO: Optional input video stream tracked boxes are rendered over +// (Required if VIZ is specified). +// FEATURES: Input feature points (std::vector) in the original +// pixel space. +// DESCRIPTORS: Input feature descriptors (std::vector). Actual feature +// dimension needs to be specified in detector_options. +// IMAGE_SIZE: Input image dimension. +// TRACKED_BOXES : input box tracking result (proto TimedBoxProtoList) from +// BoxTrackerCalculator. +// ADD_INDEX: Optional std::string containing binary format proto of type +// BoxDetectorIndex. Used for adding target index to the detector +// search index during runtime. +// CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended +// to be used with SyncSetInputStreamHandler. +// REACQ_SWITCH: Optional bool for swithcing on and off reacquisition +// functionality. User should initialize a graph with box detector +// calculator and be able to switch it on and off in runtime. +// +// Output streams: +// VIZ: Optional output video stream with rendered box positions +// (requires VIDEO to be present) +// BOXES: Optional output stream of type TimedBoxProtoList for each lost box. +// +// Imput side packets: +// INDEX_PROTO_STRING: Optional std::string containing binary format proto of +// type +// BoxDetectorIndex. Used for initializing box_detector +// with predefined template images. +// FRAME_ALIGNMENT: Optional integer to indicate alignment_boundary for +// outputing ImageFrame in "VIZ" stream. +// Set to ImageFrame::kDefaultAlignmentBoundary for +// offline pipeline to be compatible with FFmpeg. +// Set to ImageFrame::kGlDefaultAlignmentBoundary for Apps +// to be compatible with GL renderer. +// OUTPUT_INDEX_FILENAME: File path to the output index file. + +class BoxDetectorCalculator : public CalculatorBase { + public: + ~BoxDetectorCalculator() override = default; + + static ::mediapipe::Status GetContract(CalculatorContract* cc); + + ::mediapipe::Status Open(CalculatorContext* cc) override; + ::mediapipe::Status Process(CalculatorContext* cc) override; + ::mediapipe::Status Close(CalculatorContext* cc) override; + + private: + BoxDetectorCalculatorOptions options_; + std::unique_ptr box_detector_; + bool detector_switch_ = true; + uint32 frame_alignment_ = ImageFrame::kDefaultAlignmentBoundary; + bool write_index_ = false; + int box_id_ = 0; +}; + +REGISTER_CALCULATOR(BoxDetectorCalculator); + +::mediapipe::Status BoxDetectorCalculator::GetContract(CalculatorContract* cc) { + if (cc->Inputs().HasTag("TRACKING")) { + cc->Inputs().Tag("TRACKING").Set(); + } + + if (cc->Inputs().HasTag("TRACKED_BOXES")) { + cc->Inputs().Tag("TRACKED_BOXES").Set(); + } + + if (cc->Inputs().HasTag("VIDEO")) { + cc->Inputs().Tag("VIDEO").Set(); + } + + if (cc->Inputs().HasTag("FEATURES")) { + RET_CHECK(cc->Inputs().HasTag("DESCRIPTORS")) + << "FEATURES and DESCRIPTORS need to be specified together."; + cc->Inputs().Tag("FEATURES").Set>(); + } + + if (cc->Inputs().HasTag("DESCRIPTORS")) { + RET_CHECK(cc->Inputs().HasTag("FEATURES")) + << "FEATURES and DESCRIPTORS need to be specified together."; + cc->Inputs().Tag("DESCRIPTORS").Set>(); + } + + if (cc->Inputs().HasTag("IMAGE_SIZE")) { + cc->Inputs().Tag("IMAGE_SIZE").Set>(); + } + + if (cc->Inputs().HasTag("ADD_INDEX")) { + cc->Inputs().Tag("ADD_INDEX").Set(); + } + + if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) { + cc->Inputs().Tag("CANCEL_OBJECT_ID").Set(); + } + + if (cc->Inputs().HasTag("REACQ_SWITCH")) { + cc->Inputs().Tag("REACQ_SWITCH").Set(); + } + + if (cc->Outputs().HasTag("BOXES")) { + cc->Outputs().Tag("BOXES").Set(); + } + + if (cc->Outputs().HasTag("VIZ")) { + RET_CHECK(cc->Inputs().HasTag("VIDEO")) + << "Output stream VIZ requires VIDEO to be present."; + cc->Outputs().Tag("VIZ").Set(); + } + + if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) { + cc->InputSidePackets().Tag("INDEX_PROTO_STRING").Set(); + } + + if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) { + cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Set(); + } + + if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) { + cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Set(); + } + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status BoxDetectorCalculator::Open(CalculatorContext* cc) { + options_ = cc->Options(); + box_detector_ = BoxDetectorInterface::Create(options_.detector_options()); + + if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) { + BoxDetectorIndex predefined_index; + if (!predefined_index.ParseFromString(cc->InputSidePackets() + .Tag("INDEX_PROTO_STRING") + .Get())) { + LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING"; + } + box_detector_->AddBoxDetectorIndex(predefined_index); + } + + for (const auto& filename : options_.index_proto_filename()) { + std::string string_path; + ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(filename)); + std::string index_string; + MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string)); + BoxDetectorIndex predefined_index; + if (!predefined_index.ParseFromString(index_string)) { + LOG(FATAL) + << "failed to parse BoxDetectorIndex from index_proto_filename"; + } + box_detector_->AddBoxDetectorIndex(predefined_index); + } + + if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) { + write_index_ = true; + } + + if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) { + frame_alignment_ = cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Get(); + } + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { + const Timestamp timestamp = cc->InputTimestamp(); + const int64 timestamp_msec = timestamp.Value() / 1000; + + InputStream* cancel_object_id_stream = + cc->Inputs().HasTag("CANCEL_OBJECT_ID") + ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID")) + : nullptr; + if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) { + const int cancel_object_id = cancel_object_id_stream->Get(); + box_detector_->CancelBoxDetection(cancel_object_id); + } + + InputStream* add_index_stream = cc->Inputs().HasTag("ADD_INDEX") + ? &(cc->Inputs().Tag("ADD_INDEX")) + : nullptr; + if (add_index_stream && !add_index_stream->IsEmpty()) { + BoxDetectorIndex predefined_index; + if (!predefined_index.ParseFromString( + add_index_stream->Get())) { + LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX"; + } + box_detector_->AddBoxDetectorIndex(predefined_index); + } + + InputStream* reacq_switch_stream = cc->Inputs().HasTag("REACQ_SWITCH") + ? &(cc->Inputs().Tag("REACQ_SWITCH")) + : nullptr; + if (reacq_switch_stream && !reacq_switch_stream->IsEmpty()) { + detector_switch_ = reacq_switch_stream->Get(); + } + + if (!detector_switch_) { + return ::mediapipe::OkStatus(); + } + + InputStream* track_stream = cc->Inputs().HasTag("TRACKING") + ? &(cc->Inputs().Tag("TRACKING")) + : nullptr; + InputStream* video_stream = + cc->Inputs().HasTag("VIDEO") ? &(cc->Inputs().Tag("VIDEO")) : nullptr; + InputStream* feature_stream = cc->Inputs().HasTag("FEATURES") + ? &(cc->Inputs().Tag("FEATURES")) + : nullptr; + InputStream* descriptor_stream = cc->Inputs().HasTag("DESCRIPTORS") + ? &(cc->Inputs().Tag("DESCRIPTORS")) + : nullptr; + + CHECK(track_stream != nullptr || video_stream != nullptr || + (feature_stream != nullptr && descriptor_stream != nullptr)) + << "One and only one of {tracking_data, input image frame, " + "feature/descriptor} need to be valid."; + + InputStream* tracked_boxes_stream = cc->Inputs().HasTag("TRACKED_BOXES") + ? &(cc->Inputs().Tag("TRACKED_BOXES")) + : nullptr; + std::unique_ptr detected_boxes(new TimedBoxProtoList()); + + if (track_stream != nullptr) { + // Detect from tracking data + if (track_stream->IsEmpty()) { + return ::mediapipe::OkStatus(); + } + + const TrackingData& tracking_data = track_stream->Get(); + + CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed."; + + const TimedBoxProtoList tracked_boxes = + tracked_boxes_stream->Get(); + + box_detector_->DetectAndAddBox(tracking_data, tracked_boxes, timestamp_msec, + detected_boxes.get()); + } else if (video_stream != nullptr) { + // Detect from input frame + if (video_stream->IsEmpty()) { + return ::mediapipe::OkStatus(); + } + + TimedBoxProtoList tracked_boxes; + if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) { + tracked_boxes = tracked_boxes_stream->Get(); + } + + // Just directly pass along the image frame data as-is for detection; we + // don't need to worry about conforming to a specific alignment here. + const cv::Mat input_view = + formats::MatView(&video_stream->Get()); + box_detector_->DetectAndAddBox(input_view, tracked_boxes, timestamp_msec, + detected_boxes.get()); + } else { + if (feature_stream->IsEmpty() || descriptor_stream->IsEmpty()) { + return ::mediapipe::OkStatus(); + } + + const auto& image_size = + cc->Inputs().Tag("IMAGE_SIZE").Get>(); + float inv_scale = 1.0f / std::max(image_size.first, image_size.second); + + TimedBoxProtoList tracked_boxes; + if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) { + tracked_boxes = tracked_boxes_stream->Get(); + } else if (write_index_) { + auto* box_ptr = tracked_boxes.add_box(); + box_ptr->set_id(box_id_); + box_ptr->set_reacquisition(true); + box_ptr->set_aspect_ratio((float)image_size.first / + (float)image_size.second); + + box_ptr->mutable_quad()->add_vertices(0); + box_ptr->mutable_quad()->add_vertices(0); + + box_ptr->mutable_quad()->add_vertices(0); + box_ptr->mutable_quad()->add_vertices(1); + + box_ptr->mutable_quad()->add_vertices(1); + box_ptr->mutable_quad()->add_vertices(1); + + box_ptr->mutable_quad()->add_vertices(1); + box_ptr->mutable_quad()->add_vertices(0); + + ++box_id_; + } + + const auto& features = feature_stream->Get>(); + const int feature_size = features.size(); + std::vector features_vec(feature_size); + + const auto& descriptors = descriptor_stream->Get>(); + const int dims = options_.detector_options().descriptor_dims(); + CHECK_GE(descriptors.size(), feature_size * dims); + cv::Mat descriptors_mat(feature_size, dims, CV_32F); + for (int j = 0; j < feature_size; ++j) { + features_vec[j].Set(features[j].pt.x * inv_scale, + features[j].pt.y * inv_scale); + for (int i = 0; i < dims; ++i) { + descriptors_mat.at(j, i) = descriptors[j * dims + i]; + } + } + + box_detector_->DetectAndAddBoxFromFeatures( + features_vec, descriptors_mat, tracked_boxes, timestamp_msec, + image_size.first * inv_scale, image_size.second * inv_scale, + detected_boxes.get()); + } + + if (cc->Outputs().HasTag("VIZ")) { + cv::Mat viz_view; + std::unique_ptr viz_frame; + if (video_stream != nullptr && !video_stream->IsEmpty()) { + viz_frame = absl::make_unique(); + viz_frame->CopyFrom(video_stream->Get(), frame_alignment_); + viz_view = formats::MatView(viz_frame.get()); + } + for (const auto& box : detected_boxes->box()) { + RenderBox(box, &viz_view); + } + cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp); + } + + if (cc->Outputs().HasTag("BOXES")) { + cc->Outputs().Tag("BOXES").Add(detected_boxes.release(), timestamp); + } + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status BoxDetectorCalculator::Close(CalculatorContext* cc) { + if (write_index_) { + BoxDetectorIndex index = box_detector_->ObtainBoxDetectorIndex(); + MEDIAPIPE_CHECK_OK(mediapipe::file::SetContents( + cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Get(), + index.SerializeAsString())); + } + return ::mediapipe::OkStatus(); +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/video/box_detector_calculator.proto b/mediapipe/calculators/video/box_detector_calculator.proto new file mode 100644 index 000000000..f0eb42f16 --- /dev/null +++ b/mediapipe/calculators/video/box_detector_calculator.proto @@ -0,0 +1,31 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; +import "mediapipe/util/tracking/box_detector.proto"; + +message BoxDetectorCalculatorOptions { + extend CalculatorOptions { + optional BoxDetectorCalculatorOptions ext = 289746530; + } + + optional BoxDetectorOptions detector_options = 1; + + // File path to the template index files. + repeated string index_proto_filename = 2; +} diff --git a/mediapipe/calculators/video/tracked_detection_manager_calculator.cc b/mediapipe/calculators/video/tracked_detection_manager_calculator.cc index 749647d17..206711f43 100644 --- a/mediapipe/calculators/video/tracked_detection_manager_calculator.cc +++ b/mediapipe/calculators/video/tracked_detection_manager_calculator.cc @@ -18,6 +18,7 @@ #include #include "absl/container/node_hash_map.h" +#include "mediapipe/calculators/video/tracked_detection_manager_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/location_data.pb.h" @@ -139,6 +140,7 @@ Detection GetAxisAlignedDetectionFromTrackedDetection( class TrackedDetectionManagerCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc); + ::mediapipe::Status Open(CalculatorContext* cc) override; ::mediapipe::Status Process(CalculatorContext* cc) override; @@ -184,6 +186,15 @@ REGISTER_CALCULATOR(TrackedDetectionManagerCalculator); return ::mediapipe::OkStatus(); } +::mediapipe::Status TrackedDetectionManagerCalculator::Open( + CalculatorContext* cc) { + mediapipe::TrackedDetectionManagerCalculatorOptions options = + cc->Options(); + tracked_detection_manager_.SetConfig( + options.tracked_detection_manager_options()); + return ::mediapipe::OkStatus(); +} + ::mediapipe::Status TrackedDetectionManagerCalculator::Process( CalculatorContext* cc) { if (cc->Inputs().HasTag("TRACKING_BOXES")) { diff --git a/mediapipe/calculators/video/tracked_detection_manager_calculator.proto b/mediapipe/calculators/video/tracked_detection_manager_calculator.proto new file mode 100644 index 000000000..446242588 --- /dev/null +++ b/mediapipe/calculators/video/tracked_detection_manager_calculator.proto @@ -0,0 +1,28 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; +import "mediapipe/util/tracking/tracked_detection_manager_config.proto"; + +message TrackedDetectionManagerCalculatorOptions { + extend CalculatorOptions { + optional TrackedDetectionManagerCalculatorOptions ext = 301970230; + } + + optional TrackedDetectionManagerConfig tracked_detection_manager_options = 1; +} diff --git a/mediapipe/docs/face_detection_desktop.md b/mediapipe/docs/face_detection_desktop.md index b6be25b3f..b5f5f2cf4 100644 --- a/mediapipe/docs/face_detection_desktop.md +++ b/mediapipe/docs/face_detection_desktop.md @@ -51,7 +51,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run: ```bash # Video from webcam running on desktop GPU # This works only for linux currently -$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ +$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \ mediapipe/examples/desktop/face_detection:face_detection_gpu # It should print: diff --git a/mediapipe/docs/gpu.md b/mediapipe/docs/gpu.md index 630d53cf4..01b4f35b7 100644 --- a/mediapipe/docs/gpu.md +++ b/mediapipe/docs/gpu.md @@ -190,10 +190,10 @@ within the MediaPipe framework: bazel build --define MEDIAPIPE_DISABLE_GPU=1 # to enable full GPU support (OpenGL ES 3.1+ & Metal) -bazel build --copt -DMESA_EGL_NO_X11_HEADERS +bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 # to enable only OpenGL ES 3.0 and below (no GLES 3.1+ features) -bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE +bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE ``` Note *MEDIAPIPE_DISABLE_GL_COMPUTE* is automatically defined on all Apple diff --git a/mediapipe/docs/hair_segmentation_desktop.md b/mediapipe/docs/hair_segmentation_desktop.md index 91dbbd595..1161fd34f 100644 --- a/mediapipe/docs/hair_segmentation_desktop.md +++ b/mediapipe/docs/hair_segmentation_desktop.md @@ -26,7 +26,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run: ```bash # Video from webcam running on desktop GPU # This works only for linux currently -$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ +$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \ mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu # It should print: diff --git a/mediapipe/docs/hand_tracking_desktop.md b/mediapipe/docs/hand_tracking_desktop.md index 32ecd4038..58cca4470 100644 --- a/mediapipe/docs/hand_tracking_desktop.md +++ b/mediapipe/docs/hand_tracking_desktop.md @@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run: ```bash # Video from webcam running on desktop GPU # This works only for linux currently -$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ +$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \ mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu # It should print: diff --git a/mediapipe/docs/install.md b/mediapipe/docs/install.md index c7b227d76..f78392c2c 100644 --- a/mediapipe/docs/install.md +++ b/mediapipe/docs/install.md @@ -40,12 +40,11 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 1.0.0 and 1.2.1). +2. Install Bazel. Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) - to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+ - yet. + to install Bazel 2.0 or higher. 3. Install OpenCV and FFmpeg. @@ -111,7 +110,7 @@ To build and run iOS apps: # To compile with GPU support, replace --define MEDIAPIPE_DISABLE_GPU=1 # with - --copt -DMESA_EGL_NO_X11_HEADERS + --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 # when building GPU examples. ``` @@ -125,7 +124,7 @@ To build and run iOS apps: mediapipe/examples/desktop/hello_world:hello_world # If you are running on Linux desktop with GPU support enabled (via mesa drivers) - $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS \ + $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \ mediapipe/examples/desktop/hello_world:hello_world # Should print: @@ -152,12 +151,11 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 1.0.0 and 1.2.1). +2. Install Bazel. Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html) - to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+ - yet. + to install Bazel 2.0 or higher. 3. Install OpenCV. @@ -241,23 +239,18 @@ To build and run iOS apps: $ cd mediapipe ``` -3. Install Bazel (version between 1.0.0 and 1.1.0). +3. Install Bazel. - Option 1. Use package manager tool to install Bazel 1.1.0 + Option 1. Use package manager tool to install Bazel ```bash - # If Bazel 1.1.0+ was installed. - $ brew uninstall bazel - # Install Bazel 1.1.0 - $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb - $ brew link bazel + $ brew install bazel # Run 'bazel version' to check version of bazel ``` Option 2. Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x) - to install any version of Bazel manually. Note that MediaPipe doesn't - support Bazel 1.1.0+ on macOS yet. + to install Bazel 2.0 or higher. 4. Install OpenCV and FFmpeg. @@ -391,18 +384,18 @@ cameras. Alternatively, you use a video file as input. username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk ``` -5. Install Bazel (version between 1.0.0 and 1.2.1). +5. Install Bazel. ```bash username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \ - https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \ - sudo mkdir -p /usr/local/bazel/1.0.0 && \ - chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \ - sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \ - source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash + https://storage.googleapis.com/bazel/2.0.0/release/bazel-2.0.0-installer-linux-x86_64.sh && \ + sudo mkdir -p /usr/local/bazel/2.0.0 && \ + chmod 755 bazel-2.0.0-installer-linux-x86_64.sh && \ + sudo ./bazel-2.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/2.0.0 && \ + source /usr/local/bazel/2.0.0/lib/bazel/bin/bazel-complete.bash - username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \ - alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel' + username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/2.0.0/lib/bazel/bin/bazel version && \ + alias bazel='/usr/local/bazel/2.0.0/lib/bazel/bin/bazel' ``` 6. Checkout MediaPipe repository. diff --git a/mediapipe/docs/media_sequence.md b/mediapipe/docs/media_sequence.md index 239cff4d2..e248aa5e4 100644 --- a/mediapipe/docs/media_sequence.md +++ b/mediapipe/docs/media_sequence.md @@ -89,11 +89,11 @@ process new data sets, in the documentation of dataset = d.as_dataset('test') # implement additional processing and batching here dataset_output = dataset.make_one_shot_iterator().get_next() - images = dataset_output=['images'] - labels = dataset_output=['labels'] + images = dataset_output['images'] + labels = dataset_output['labels'] with tf.Session() as sess: - images_, labels_ = sess.run(images, labels) + images_, labels_ = sess.run([images, labels]) print('The shape of images_ is %s' % str(images_.shape)) print('The shape of labels_ is %s' % str(labels_.shape)) ``` diff --git a/mediapipe/docs/mediapipe_ios_setup.md b/mediapipe/docs/mediapipe_ios_setup.md index 71a65d4b8..8f8d2880c 100644 --- a/mediapipe/docs/mediapipe_ios_setup.md +++ b/mediapipe/docs/mediapipe_ios_setup.md @@ -76,6 +76,10 @@ in the next section. 2. Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app. + Important: If Tulsi displays an error saying "Bazel could not be found", + press the "Bazel..." button in the Packages tab and select the `bazel` + executable in your homebrew `/bin/` directory. + 3. Select the MediaPipe config in the Configs tab, then press the Generate button below. You will be asked for a location to save the Xcode project. Once the project is generated, it will be opened in Xcode. diff --git a/mediapipe/docs/multi_hand_tracking_desktop.md b/mediapipe/docs/multi_hand_tracking_desktop.md index 3369388ee..b6646d074 100644 --- a/mediapipe/docs/multi_hand_tracking_desktop.md +++ b/mediapipe/docs/multi_hand_tracking_desktop.md @@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run: ```bash # Video from webcam running on desktop GPU # This works only for linux currently -$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \ +$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \ mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu # It should print: diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD index 94a6101cd..e7549abd0 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD @@ -60,8 +60,8 @@ android_library( "//third_party:androidx_appcompat", "//third_party:androidx_constraint_layout", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD index c447095fd..773a83e77 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD @@ -64,10 +64,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD index 7e6e3b278..39b8fa315 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD @@ -64,10 +64,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD index 609840e1b..b0dd72419 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD @@ -63,10 +63,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD index c774e1ce5..cec7efa01 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD @@ -64,10 +64,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD index ee1c52486..27701a17d 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD @@ -85,10 +85,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD index e2c671423..c70c347bc 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD @@ -85,10 +85,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD index 1f1f203b4..37e65fb7c 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD @@ -97,10 +97,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD index 1b990e9a4..7efecb41c 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD @@ -64,10 +64,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD index 94f59c126..99050e438 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD @@ -65,10 +65,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD index be0fc0cf2..01d3f40c5 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD @@ -64,9 +64,9 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", "//third_party:opencv", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/examples/coral/Dockerfile b/mediapipe/examples/coral/Dockerfile index a2c2a153f..28c76a5b4 100644 --- a/mediapipe/examples/coral/Dockerfile +++ b/mediapipe/examples/coral/Dockerfile @@ -35,10 +35,8 @@ RUN dpkg --add-architecture arm64 RUN apt-get update && apt-get install -y \ build-essential \ crossbuild-essential-arm64 \ - libusb-1.0-0-dev \ libusb-1.0-0-dev:arm64 \ - zlib1g-dev \ - zlib1g-dev:arm64 \ + zlibc:arm64 \ pkg-config \ zip \ unzip \ @@ -62,8 +60,8 @@ RUN pip3 install six COPY . /mediapipe/ # Install bazel - -ARG BAZEL_VERSION=1.1.0 +# Please match the current MediaPipe Bazel requirements according to docs. +ARG BAZEL_VERSION=2.0.0 RUN mkdir /bazel && \ wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ diff --git a/mediapipe/examples/coral/WORKSPACE b/mediapipe/examples/coral/WORKSPACE deleted file mode 100644 index 3afda1a86..000000000 --- a/mediapipe/examples/coral/WORKSPACE +++ /dev/null @@ -1,356 +0,0 @@ -workspace(name = "mediapipe") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -skylib_version = "0.8.0" -http_archive( - name = "bazel_skylib", - type = "tar.gz", - url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version), - sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", -) -load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "1.0.0", - maximum_bazel_version = "1.2.1") - - -# ABSL cpp library lts_2020_02_25 -http_archive( - name = "com_google_absl", - urls = [ - "https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz", - ], - # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved. - patches = [ - "@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff" - ], - patch_args = [ - "-p1", - ], - strip_prefix = "abseil-cpp-20200225", - sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353" -) - -http_archive( - name = "rules_cc", - strip_prefix = "rules_cc-master", - urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"], -) - -# GoogleTest/GoogleMock framework. Used by most unit-tests. -http_archive( - name = "com_google_googletest", - urls = ["https://github.com/google/googletest/archive/master.zip"], - strip_prefix = "googletest-master", -) - -# Google Benchmark library. -http_archive( - name = "com_google_benchmark", - urls = ["https://github.com/google/benchmark/archive/master.zip"], - strip_prefix = "benchmark-master", - build_file = "@//third_party:benchmark.BUILD", -) - -# gflags needed by glog -http_archive( - name = "com_github_gflags_gflags", - sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe", - strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a", - urls = [ - "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz", - "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz", - ], -) - -# glog -http_archive( - name = "com_github_glog_glog", - url = "https://github.com/google/glog/archive/v0.3.5.zip", - sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8", - strip_prefix = "glog-0.3.5", - build_file = "@//third_party:glog.BUILD", - patches = [ - "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff" - ], - patch_args = [ - "-p1", - ], -) - -# easyexif -http_archive( - name = "easyexif", - url = "https://github.com/mayanklahiri/easyexif/archive/master.zip", - strip_prefix = "easyexif-master", - build_file = "@//third_party:easyexif.BUILD", -) - -# libyuv -http_archive( - name = "libyuv", - urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"], - build_file = "@//third_party:libyuv.BUILD", -) - -http_archive( - name = "com_google_protobuf_javalite", - sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc", - strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516", - urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"], -) - -http_archive( - name = "com_google_audio_tools", - strip_prefix = "multichannel-audio-tools-master", - urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"], -) - -# Needed by TensorFlow -http_archive( - name = "io_bazel_rules_closure", - sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9", - strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df", - urls = [ - "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", # 2019-04-04 - ], -) - -# 2020-02-12 -# The last commit before TensorFlow switched to Bazel 2.0 -_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328" -_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13" -http_archive( - name = "org_tensorflow", - urls = [ - "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, - "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, - ], - # A compatibility patch - patches = [ - "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff" - ], - patch_args = [ - "-p1", - ], - strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT, - sha256 = _TENSORFLOW_SHA256, -) - -load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace") -tf_workspace(tf_repo_name = "org_tensorflow") - -http_archive( - name = "ceres_solver", - url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip", - patches = [ - "@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff" - ], - patch_args = [ - "-p1", - ], - strip_prefix = "ceres-solver-1.14.0", - sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1" -) - -# Please run -# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \ -# libopencv-calib3d-dev libopencv-features2d-dev \ -# libopencv-imgproc-dev libopencv-video-dev -new_local_repository( - name = "linux_opencv", - build_file = "@//third_party:opencv_linux.BUILD", - path = "/usr", -) - -new_local_repository( - name = "linux_ffmpeg", - build_file = "@//third_party:ffmpeg_linux.BUILD", - path = "/usr" -) - -# Please run $ brew install opencv@3 -new_local_repository( - name = "macos_opencv", - build_file = "@//third_party:opencv_macos.BUILD", - path = "/usr", -) - -new_local_repository( - name = "macos_ffmpeg", - build_file = "@//third_party:ffmpeg_macos.BUILD", - path = "/usr", -) - -http_archive( - name = "android_opencv", - build_file = "@//third_party:opencv_android.BUILD", - strip_prefix = "OpenCV-android-sdk", - type = "zip", - url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip", -) - -# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will -# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app. -# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with -# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'. -http_archive( - name = "ios_opencv", - sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2", - build_file = "@//third_party:opencv_ios.BUILD", - type = "zip", - url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip", -) - -RULES_JVM_EXTERNAL_TAG = "2.2" -RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6" - -http_archive( - name = "rules_jvm_external", - strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, - sha256 = RULES_JVM_EXTERNAL_SHA, - url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, -) - -load("@rules_jvm_external//:defs.bzl", "maven_install") - -maven_install( - artifacts = [ - "androidx.annotation:annotation:aar:1.1.0", - "androidx.appcompat:appcompat:aar:1.1.0-rc01", - "androidx.camera:camera-core:aar:1.0.0-alpha06", - "androidx.camera:camera-camera2:aar:1.0.0-alpha06", - "androidx.constraintlayout:constraintlayout:aar:1.1.3", - "androidx.core:core:aar:1.1.0-rc03", - "androidx.legacy:legacy-support-v4:aar:1.0.0", - "androidx.recyclerview:recyclerview:aar:1.1.0-beta02", - "com.google.android.material:material:aar:1.0.0-rc01", - ], - repositories = [ - "https://dl.google.com/dl/android/maven2", - "https://repo1.maven.org/maven2", - ], -) - -maven_server( - name = "google_server", - url = "https://dl.google.com/dl/android/maven2", -) - -maven_jar( - name = "androidx_lifecycle", - artifact = "androidx.lifecycle:lifecycle-common:2.0.0", - sha1 = "e070ffae07452331bc5684734fce6831d531785c", - server = "google_server", -) - -maven_jar( - name = "androidx_concurrent_futures", - artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03", - sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae", - server = "google_server", -) - -maven_jar( - name = "com_google_guava_android", - artifact = "com.google.guava:guava:27.0.1-android", - sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d", -) - -maven_jar( - name = "com_google_common_flogger", - artifact = "com.google.flogger:flogger:0.3.1", - sha1 = "585030fe1ec709760cbef997a459729fb965df0e", -) - -maven_jar( - name = "com_google_common_flogger_system_backend", - artifact = "com.google.flogger:flogger-system-backend:0.3.1", - sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9", -) - -maven_jar( - name = "com_google_code_findbugs", - artifact = "com.google.code.findbugs:jsr305:3.0.2", - sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d", -) - -# You may run setup_android.sh to install Android SDK and NDK. -android_ndk_repository( - name = "androidndk", -) - -android_sdk_repository( - name = "androidsdk", -) - -# iOS basic build deps. - -http_archive( - name = "build_bazel_rules_apple", - sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", - strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", - url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", -) - -load( - "@build_bazel_rules_apple//apple:repositories.bzl", - "apple_rules_dependencies", -) - -apple_rules_dependencies() - -load( - "@build_bazel_rules_swift//swift:repositories.bzl", - "swift_rules_dependencies", -) - -swift_rules_dependencies() - -load( - "@build_bazel_apple_support//lib:repositories.bzl", - "apple_support_dependencies", -) - -apple_support_dependencies() - -# More iOS deps. - -http_archive( - name = "google_toolbox_for_mac", - url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip", - sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc", - strip_prefix = "google-toolbox-for-mac-2.2.1", - build_file = "@//third_party:google_toolbox_for_mac.BUILD", -) - -### Coral ### - -#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}') -#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}') -# Oct 2019 -#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229 -#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb -http_archive( - name = "coral_crosstool", - sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb", - strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229", - urls = [ - "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz", - ], -) -load("@coral_crosstool//:configure.bzl", "cc_crosstool") -cc_crosstool(name = "crosstool") - -# EdgeTPU -new_local_repository( - name = "edgetpu", - path = "/edgetpu/libedgetpu", - build_file = "/edgetpu/libedgetpu/BUILD" -) -new_local_repository( - name = "libedgetpu", - path = "/usr/lib/aarch64-linux-gnu", - build_file = "/edgetpu/libedgetpu/BUILD" -) diff --git a/mediapipe/examples/coral/WORKSPACE.coral b/mediapipe/examples/coral/WORKSPACE.coral new file mode 100644 index 000000000..f990ef2f5 --- /dev/null +++ b/mediapipe/examples/coral/WORKSPACE.coral @@ -0,0 +1,30 @@ + +### Coral additions to MediaPipe WORKSPACE ### + +#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}') +#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}') +# Oct 2019 +#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229 +#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb +http_archive( + name = "coral_crosstool", + sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb", + strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229", + urls = [ + "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz", + ], +) +load("@coral_crosstool//:configure.bzl", "cc_crosstool") +cc_crosstool(name = "crosstool") + +# EdgeTPU +new_local_repository( + name = "edgetpu", + path = "/edgetpu/libedgetpu", + build_file = "/edgetpu/libedgetpu/BUILD" +) +new_local_repository( + name = "libedgetpu", + path = "/usr/lib/aarch64-linux-gnu", + build_file = "/edgetpu/libedgetpu/BUILD" +) diff --git a/mediapipe/examples/coral/setup.sh b/mediapipe/examples/coral/setup.sh index dabf4c7dd..884a7a10d 100755 --- a/mediapipe/examples/coral/setup.sh +++ b/mediapipe/examples/coral/setup.sh @@ -10,12 +10,25 @@ sleep 3 mkdir -p opencv32_arm64_libs +# prepare docker aux script cp mediapipe/examples/coral/update_sources.sh update_sources.sh chmod +x update_sources.sh +# backup non-coral Dockerfile mv Dockerfile Dockerfile.orig cp mediapipe/examples/coral/Dockerfile Dockerfile +# backup non-coral workspace cp WORKSPACE WORKSPACE.orig -cp mediapipe/examples/coral/WORKSPACE WORKSPACE +# create temps +cp WORKSPACE WORKSPACE.1 +cp mediapipe/examples/coral/WORKSPACE.coral WORKSPACE.2 + +# merge (shell decides concat order, unless numbered appropriately) +cat WORKSPACE.1 WORKSPACE.2 > WORKSPACE + +# cleanup +rm WORKSPACE.1 WORKSPACE.2 + +echo 'done' diff --git a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc index 63dc9a198..8d8e2570a 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc @@ -85,7 +85,7 @@ REGISTER_CALCULATOR(ShotBoundaryCalculator); void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image, cv::Mat* image_histogram) { cv::Mat equalized_image; - cv::cvtColor(image.clone(), equalized_image, CV_RGB2GRAY); + cv::cvtColor(image.clone(), equalized_image, cv::COLOR_RGB2GRAY); double min, max; cv::minMaxLoc(equalized_image, &min, &max); diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 4ab28fbd2..292756cc2 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1496,13 +1496,17 @@ cc_test( "//mediapipe/framework/port:status", "//mediapipe/framework/stream_handler:barrier_input_stream_handler", "//mediapipe/framework/stream_handler:early_close_input_stream_handler", + "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/stream_handler:immediate_input_stream_handler", "//mediapipe/framework/stream_handler:mux_input_stream_handler", + "//mediapipe/framework/stream_handler:sync_set_input_stream_handler", + "//mediapipe/framework/stream_handler:timestamp_align_input_stream_handler", "//mediapipe/framework/tool:sink", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", ], ) diff --git a/mediapipe/framework/calculator_graph_test.cc b/mediapipe/framework/calculator_graph_test.cc index f45fd5175..6f749100c 100644 --- a/mediapipe/framework/calculator_graph_test.cc +++ b/mediapipe/framework/calculator_graph_test.cc @@ -29,6 +29,7 @@ #include "absl/memory/memory.h" #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" @@ -4558,5 +4559,68 @@ TEST(CalculatorGraph, SimpleMuxCalculatorWithCustomInputStreamHandler) { testing::HasSubstr("ImmediateInputStreamHandler class comment"))); } +void DoTestMultipleGraphRuns(absl::string_view input_stream_handler, + bool select_packet) { + std::string graph_proto = absl::StrFormat(R"( + input_stream: 'input' + input_stream: 'select' + node { + calculator: 'PassThroughCalculator' + input_stream: 'input' + input_stream: 'select' + input_stream_handler { + input_stream_handler: "%s" + } + output_stream: 'output' + output_stream: 'select_out' + } + )", + input_stream_handler.data()); + CalculatorGraphConfig config = + ::mediapipe::ParseTextProtoOrDie(graph_proto); + std::vector packet_dump; + tool::AddVectorSink("output", &config, &packet_dump); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(config)); + + struct Run { + Timestamp timestamp; + int value; + }; + std::vector runs = {{.timestamp = Timestamp(2000), .value = 2}, + {.timestamp = Timestamp(1000), .value = 1}}; + for (const Run& run : runs) { + MP_ASSERT_OK(graph.StartRun({})); + + if (select_packet) { + MP_EXPECT_OK(graph.AddPacketToInputStream( + "select", MakePacket(0).At(run.timestamp))); + } + MP_EXPECT_OK(graph.AddPacketToInputStream( + "input", MakePacket(run.value).At(run.timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(1, packet_dump.size()); + EXPECT_EQ(run.value, packet_dump[0].Get()); + EXPECT_EQ(run.timestamp, packet_dump[0].Timestamp()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + + packet_dump.clear(); + } +} + +TEST(CalculatorGraph, MultipleRunsWithDifferentInputStreamHandlers) { + DoTestMultipleGraphRuns("BarrierInputStreamHandler", true); + DoTestMultipleGraphRuns("DefaultInputStreamHandler", true); + DoTestMultipleGraphRuns("EarlyCloseInputStreamHandler", true); + DoTestMultipleGraphRuns("FixedSizeInputStreamHandler", true); + DoTestMultipleGraphRuns("ImmediateInputStreamHandler", false); + DoTestMultipleGraphRuns("MuxInputStreamHandler", true); + DoTestMultipleGraphRuns("SyncSetInputStreamHandler", true); + DoTestMultipleGraphRuns("TimestampAlignInputStreamHandler", true); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/calculator_node.cc b/mediapipe/framework/calculator_node.cc index f69254db0..d9edb4b69 100644 --- a/mediapipe/framework/calculator_node.cc +++ b/mediapipe/framework/calculator_node.cc @@ -814,7 +814,8 @@ std::string CalculatorNode::DebugName() const { input_stream_handler_->FinalizeInputSet(input_timestamp, inputs); output_stream_handler_->PrepareOutputs(input_timestamp, outputs); - VLOG(2) << "Calling Calculator::Process() for node: " << DebugName(); + VLOG(2) << "Calling Calculator::Process() for node: " << DebugName() + << " timestamp: " << input_timestamp; if (OutputsAreConstant(calculator_context)) { // Do nothing. @@ -826,6 +827,9 @@ std::string CalculatorNode::DebugName() const { result = calculator_->Process(calculator_context); } + VLOG(2) << "Called Calculator::Process() for node: " << DebugName() + << " timestamp: " << input_timestamp; + // Removes one packet from each shard and progresses to the next input // timestamp. input_stream_handler_->ClearCurrentInputs(calculator_context); diff --git a/mediapipe/framework/calculator_profile.proto b/mediapipe/framework/calculator_profile.proto index d90d8a3df..723178ae4 100644 --- a/mediapipe/framework/calculator_profile.proto +++ b/mediapipe/framework/calculator_profile.proto @@ -113,7 +113,10 @@ message GraphTrace { optional int32 stream_id = 4; // The address of the packet contents. - optional int64 packet_id = 5; + optional int64 packet_id = 5 [deprecated = true]; + + // Data describing the event, such as the packet contents. + optional int64 event_data = 6; } // The kind of event recorded. @@ -133,6 +136,7 @@ message GraphTrace { DSP_TASK = 12; TPU_TASK = 13; GPU_CALIBRATION = 14; + PACKET_QUEUED = 15; } // The timing for one packet set being processed at one caclulator node. diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 97e651f9e..3d4b71cd7 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -13,7 +13,7 @@ # limitations under the License. # -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") package( default_visibility = ["//visibility:private"], @@ -24,95 +24,45 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -proto_library( +mediapipe_proto_library( name = "detection_proto", srcs = ["detection.proto"], visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats:location_data_proto"], ) -proto_library( +mediapipe_proto_library( name = "classification_proto", srcs = ["classification.proto"], visibility = ["//visibility:public"], ) -proto_library( +mediapipe_proto_library( name = "image_format_proto", srcs = ["image_format.proto"], visibility = ["//visibility:public"], ) -proto_library( +mediapipe_proto_library( name = "matrix_data_proto", srcs = ["matrix_data.proto"], visibility = ["//visibility:public"], ) -proto_library( +mediapipe_proto_library( name = "location_data_proto", srcs = ["location_data.proto"], + portable_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"], visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"], ) -proto_library( +mediapipe_proto_library( name = "time_series_header_proto", srcs = ["time_series_header.proto"], visibility = ["//visibility:public"], ) -mediapipe_cc_proto_library( - name = "detection_cc_proto", - srcs = ["detection.proto"], - cc_deps = [":location_data_cc_proto"], - visibility = ["//visibility:public"], - deps = [":detection_proto"], -) - -java_lite_proto_library( - name = "detection_java_proto_lite", - strict_deps = 0, - visibility = ["//mediapipe:__subpackages__"], - deps = [":detection_proto"], -) - -mediapipe_cc_proto_library( - name = "classification_cc_proto", - srcs = ["classification.proto"], - visibility = ["//visibility:public"], - deps = [":classification_proto"], -) - -mediapipe_cc_proto_library( - name = "image_format_cc_proto", - srcs = ["image_format.proto"], - visibility = ["//visibility:public"], - deps = [":image_format_proto"], -) - -mediapipe_cc_proto_library( - name = "matrix_data_cc_proto", - srcs = ["matrix_data.proto"], - visibility = ["//visibility:public"], - deps = [":matrix_data_proto"], -) - -mediapipe_cc_proto_library( - name = "location_data_cc_proto", - srcs = ["location_data.proto"], - cc_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"], - visibility = ["//visibility:public"], - deps = [":location_data_proto"], -) - -mediapipe_cc_proto_library( - name = "time_series_header_cc_proto", - srcs = ["time_series_header.proto"], - visibility = ["//visibility:public"], - deps = [":time_series_header_proto"], -) - cc_library( name = "deleting_file", srcs = ["deleting_file.cc"], @@ -245,33 +195,14 @@ cc_test( ], ) -proto_library( +mediapipe_proto_library( name = "rect_proto", srcs = ["rect.proto"], - visibility = [ - "//visibility:public", - ], + visibility = ["//visibility:public"], + deps = ["//mediapipe/framework/formats:location_data_proto"], ) -mediapipe_cc_proto_library( - name = "rect_cc_proto", - srcs = ["rect.proto"], - visibility = [ - "//visibility:public", - ], - deps = [":rect_proto"], -) - -java_lite_proto_library( - name = "rect_java_proto_lite", - strict_deps = 0, - visibility = [ - "//mediapipe:__subpackages__", - ], - deps = [":rect_proto"], -) - -proto_library( +mediapipe_proto_library( name = "landmark_proto", srcs = ["landmark.proto"], visibility = [ @@ -279,22 +210,6 @@ proto_library( ], ) -mediapipe_cc_proto_library( - name = "landmark_cc_proto", - srcs = ["landmark.proto"], - visibility = ["//visibility:public"], - deps = [":landmark_proto"], -) - -java_lite_proto_library( - name = "landmark_java_proto_lite", - strict_deps = 0, - visibility = [ - "//mediapipe:__subpackages__", - ], - deps = [":landmark_proto"], -) - # Expose the proto source files for building mediapipe AAR. filegroup( name = "protos_src", diff --git a/mediapipe/framework/formats/annotation/BUILD b/mediapipe/framework/formats/annotation/BUILD index d501901a7..a754b0250 100644 --- a/mediapipe/framework/formats/annotation/BUILD +++ b/mediapipe/framework/formats/annotation/BUILD @@ -14,42 +14,28 @@ # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +package(default_visibility = ["//visibility:private"]) + licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -package(default_visibility = ["//visibility:private"]) - -proto_library( +mediapipe_proto_library( name = "locus_proto", srcs = ["locus.proto"], + portable_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"], visibility = ["//visibility:public"], deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"], ) -proto_library( +mediapipe_proto_library( name = "rasterization_proto", srcs = ["rasterization.proto"], visibility = ["//visibility:public"], ) -mediapipe_cc_proto_library( - name = "locus_cc_proto", - srcs = ["locus.proto"], - cc_deps = [":rasterization_cc_proto"], - visibility = ["//visibility:public"], - deps = [":locus_proto"], -) - -mediapipe_cc_proto_library( - name = "rasterization_cc_proto", - srcs = ["rasterization.proto"], - visibility = ["//visibility:public"], - deps = [":rasterization_proto"], -) - # Expose the proto source files for building mediapipe AAR. filegroup( name = "protos_src", diff --git a/mediapipe/framework/input_stream_handler.cc b/mediapipe/framework/input_stream_handler.cc index 0d6965056..13a2adc8a 100644 --- a/mediapipe/framework/input_stream_handler.cc +++ b/mediapipe/framework/input_stream_handler.cc @@ -230,8 +230,36 @@ void InputStreamHandler::FinalizeInputSet(Timestamp timestamp, } } +// Returns the default CalculatorContext. +CalculatorContext* GetCalculatorContext(CalculatorContextManager* manager) { + return (manager && manager->HasDefaultCalculatorContext()) + ? manager->GetDefaultCalculatorContext() + : nullptr; +} + +// Logs the current queue size of an input stream. +void LogQueuedPackets(CalculatorContext* context, InputStreamManager* stream, + Packet queue_tail) { + if (context) { + TraceEvent event = TraceEvent(TraceEvent::PACKET_QUEUED) + .set_node_id(context->NodeId()) + .set_input_ts(queue_tail.Timestamp()) + .set_stream_id(&stream->Name()) + .set_event_data(stream->QueueSize() + 1); + ::mediapipe::LogEvent(context->GetProfilingContext(), + event.set_packet_ts(queue_tail.Timestamp())); + Packet queue_head = stream->QueueHead(); + if (!queue_head.IsEmpty()) { + ::mediapipe::LogEvent(context->GetProfilingContext(), + event.set_packet_ts(queue_head.Timestamp())); + } + } +} + void InputStreamHandler::AddPackets(CollectionItemId id, const std::list& packets) { + LogQueuedPackets(GetCalculatorContext(calculator_context_manager_), + input_stream_managers_.Get(id), packets.back()); bool notify = false; ::mediapipe::Status result = input_stream_managers_.Get(id)->AddPackets(packets, ¬ify); @@ -245,6 +273,8 @@ void InputStreamHandler::AddPackets(CollectionItemId id, void InputStreamHandler::MovePackets(CollectionItemId id, std::list* packets) { + LogQueuedPackets(GetCalculatorContext(calculator_context_manager_), + input_stream_managers_.Get(id), packets->back()); bool notify = false; ::mediapipe::Status result = input_stream_managers_.Get(id)->MovePackets(packets, ¬ify); @@ -307,6 +337,8 @@ SyncSet::SyncSet(InputStreamHandler* input_stream_handler, : input_stream_handler_(input_stream_handler), stream_ids_(std::move(stream_ids)) {} +void SyncSet::PrepareForRun() { last_processed_ts_ = Timestamp::Unset(); } + NodeReadiness SyncSet::GetReadiness(Timestamp* min_stream_timestamp) { Timestamp min_bound = Timestamp::Done(); Timestamp min_packet = Timestamp::Done(); diff --git a/mediapipe/framework/input_stream_handler.h b/mediapipe/framework/input_stream_handler.h index 8d8b5a716..aa9f44285 100644 --- a/mediapipe/framework/input_stream_handler.h +++ b/mediapipe/framework/input_stream_handler.h @@ -201,6 +201,9 @@ class InputStreamHandler { SyncSet(InputStreamHandler* input_stream_handler, std::vector stream_ids); + // Reinitializes this SyncSet before each CalculatorGraph run. + void PrepareForRun(); + // Answers whether this stream is ready for Process or Close. NodeReadiness GetReadiness(Timestamp* min_stream_timestamp); diff --git a/mediapipe/framework/input_stream_manager.cc b/mediapipe/framework/input_stream_manager.cc index 5377790c6..38e6dca39 100644 --- a/mediapipe/framework/input_stream_manager.cc +++ b/mediapipe/framework/input_stream_manager.cc @@ -152,7 +152,7 @@ template // If the caller is MovePackets(), packet's underlying holder should be // transferred into queue_. Otherwise, queue_ keeps a copy of the packet. ++num_packets_added_; - VLOG(2) << "Input stream:" << name_ + VLOG(3) << "Input stream:" << name_ << " has added packet at time: " << packet.Timestamp(); if (std::is_const< typename std::remove_reference::type>::value) { @@ -163,15 +163,15 @@ template } queue_became_full = (!was_queue_full && max_queue_size_ != -1 && queue_.size() >= max_queue_size_); - VLOG_IF(2, queue_.size() > 1) + VLOG_IF(3, queue_.size() > 1) << "Queue size greater than 1: stream name: " << name_ << " queue_size: " << queue_.size(); - VLOG(2) << "Input stream:" << name_ + VLOG(3) << "Input stream:" << name_ << " becomes non-empty status:" << queue_became_non_empty << " Size: " << queue_.size(); } if (queue_became_full) { - VLOG(2) << "Queue became full: " << Name(); + VLOG(3) << "Queue became full: " << Name(); becomes_full_callback_(this, &last_reported_stream_full_); } *notify = queue_became_non_empty; @@ -257,7 +257,7 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, next_timestamp_bound_ = timestamp.NextAllowedInStream(); } - VLOG(2) << "Input stream " << name_ + VLOG(3) << "Input stream " << name_ << " selecting at timestamp:" << timestamp.Value() << " next timestamp bound: " << next_timestamp_bound_; @@ -282,13 +282,13 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, ++(*num_packets_dropped); } - VLOG(2) << "Input stream removed packets:" << name_ + VLOG(3) << "Input stream removed packets:" << name_ << " Size:" << queue_.size(); queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_); *stream_is_done = IsDone(); } if (queue_became_non_full) { - VLOG(2) << "Queue became non-full: " << Name(); + VLOG(3) << "Queue became non-full: " << Name(); becomes_not_full_callback_(this, &last_reported_stream_full_); } return packet; @@ -302,7 +302,7 @@ Packet InputStreamManager::PopQueueHead(bool* stream_is_done) { { absl::MutexLock stream_lock(&stream_mutex_); - VLOG(2) << "Input stream " << name_ << " selecting at queue head"; + VLOG(3) << "Input stream " << name_ << " selecting at queue head"; // Check if queue is full. bool was_queue_full = @@ -315,13 +315,13 @@ Packet InputStreamManager::PopQueueHead(bool* stream_is_done) { packet = Packet(); } - VLOG(2) << "Input stream removed a packet:" << name_ + VLOG(3) << "Input stream removed a packet:" << name_ << " Size:" << queue_.size(); queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_); *stream_is_done = IsDone(); } if (queue_became_non_full) { - VLOG(2) << "Queue became non-full: " << Name(); + VLOG(3) << "Queue became non-full: " << Name(); becomes_not_full_callback_(this, &last_reported_stream_full_); } return packet; @@ -349,10 +349,10 @@ void InputStreamManager::SetMaxQueueSize(int max_queue_size) { // QueueSizeCallback is called with no mutexes held. if (!was_full && is_full) { - VLOG(2) << "Queue became full: " << Name(); + VLOG(3) << "Queue became full: " << Name(); becomes_full_callback_(this, &last_reported_stream_full_); } else if (was_full && !is_full) { - VLOG(2) << "Queue became non-full: " << Name(); + VLOG(3) << "Queue became non-full: " << Name(); becomes_not_full_callback_(this, &last_reported_stream_full_); } } @@ -382,12 +382,12 @@ void InputStreamManager::ErasePacketsEarlierThan(Timestamp timestamp) { queue_.pop_front(); } - VLOG(2) << "Input stream removed packets:" << name_ + VLOG(3) << "Input stream removed packets:" << name_ << " Size:" << queue_.size(); queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_); } if (queue_became_non_full) { - VLOG(2) << "Queue became non-full: " << Name(); + VLOG(3) << "Queue became non-full: " << Name(); becomes_not_full_callback_(this, &last_reported_stream_full_); } } diff --git a/mediapipe/framework/output_stream_manager.cc b/mediapipe/framework/output_stream_manager.cc index 25c94b349..deacf4fbb 100644 --- a/mediapipe/framework/output_stream_manager.cc +++ b/mediapipe/framework/output_stream_manager.cc @@ -164,9 +164,9 @@ void OutputStreamManager::PropagateUpdatesToMirrors( next_timestamp_bound_ = next_timestamp_bound; } std::list* packets_to_propagate = output_stream_shard->OutputQueue(); - VLOG(2) << "Output stream: " << Name() + VLOG(3) << "Output stream: " << Name() << " queue size: " << packets_to_propagate->size(); - VLOG(2) << "Output stream: " << Name() + VLOG(3) << "Output stream: " << Name() << " next timestamp: " << next_timestamp_bound; bool add_packets = !packets_to_propagate->empty(); bool set_bound = diff --git a/mediapipe/framework/packet.h b/mediapipe/framework/packet.h index e08955555..8dd7a91fb 100644 --- a/mediapipe/framework/packet.h +++ b/mediapipe/framework/packet.h @@ -540,11 +540,11 @@ const Holder* HolderBase::As() const { inline Packet::Packet(const Packet& packet) : holder_(packet.holder_), timestamp_(packet.timestamp_) { - VLOG(2) << "Using copy constructor of " << packet.DebugString(); + VLOG(4) << "Using copy constructor of " << packet.DebugString(); } inline Packet& Packet::operator=(const Packet& packet) { - VLOG(2) << "Using copy assignment operator of " << packet.DebugString(); + VLOG(4) << "Using copy assignment operator of " << packet.DebugString(); if (this != &packet) { holder_ = packet.holder_; timestamp_ = packet.timestamp_; @@ -559,11 +559,11 @@ inline ::mediapipe::StatusOr> Packet::Consume() { // Clients who use this function are responsible for ensuring that no // other thread is doing anything with this Packet. if (holder_.unique()) { - VLOG(1) << "Consuming the data of " << DebugString(); + VLOG(2) << "Consuming the data of " << DebugString(); ::mediapipe::StatusOr> release_result = holder_->As()->Release(); if (release_result.ok()) { - VLOG(1) << "Setting " << DebugString() << " to empty."; + VLOG(2) << "Setting " << DebugString() << " to empty."; holder_.reset(); } return release_result; @@ -582,11 +582,11 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( // If holder is the sole owner of the underlying data, consumes this packet. if (!holder_->HolderIsOfType>() && holder_.unique()) { - VLOG(1) << "Consuming the data of " << DebugString(); + VLOG(2) << "Consuming the data of " << DebugString(); ::mediapipe::StatusOr> release_result = holder_->As()->Release(); if (release_result.ok()) { - VLOG(1) << "Setting " << DebugString() << " to empty."; + VLOG(2) << "Setting " << DebugString() << " to empty."; holder_.reset(); } if (was_copied) { @@ -594,9 +594,9 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( } return release_result; } - VLOG(1) << "Copying the data of " << DebugString(); + VLOG(2) << "Copying the data of " << DebugString(); std::unique_ptr data_ptr = absl::make_unique(Get()); - VLOG(1) << "Setting " << DebugString() << " to empty."; + VLOG(2) << "Setting " << DebugString() << " to empty."; holder_.reset(); if (was_copied) { *was_copied = true; @@ -613,11 +613,11 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( // If holder is the sole owner of the underlying data, consumes this packet. if (!holder_->HolderIsOfType>() && holder_.unique()) { - VLOG(1) << "Consuming the data of " << DebugString(); + VLOG(2) << "Consuming the data of " << DebugString(); ::mediapipe::StatusOr> release_result = holder_->As()->Release(); if (release_result.ok()) { - VLOG(1) << "Setting " << DebugString() << " to empty."; + VLOG(2) << "Setting " << DebugString() << " to empty."; holder_.reset(); } if (was_copied) { @@ -625,7 +625,7 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( } return release_result; } - VLOG(1) << "Copying the data of " << DebugString(); + VLOG(2) << "Copying the data of " << DebugString(); const auto& original_array = Get(); // Type T is bounded array type, such as int[N] and float[M]. // The new operator creates a new bounded array. @@ -633,7 +633,7 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( // Copies bounded array data into data_ptr. std::copy(std::begin(original_array), std::end(original_array), std::begin(*data_ptr)); - VLOG(1) << "Setting " << DebugString() << " to empty."; + VLOG(2) << "Setting " << DebugString() << " to empty."; holder_.reset(); if (was_copied) { *was_copied = true; @@ -650,14 +650,14 @@ inline ::mediapipe::StatusOr> Packet::ConsumeOrCopy( } inline Packet::Packet(Packet&& packet) { - VLOG(2) << "Using move constructor of " << packet.DebugString(); + VLOG(4) << "Using move constructor of " << packet.DebugString(); holder_ = std::move(packet.holder_); timestamp_ = packet.timestamp_; packet.timestamp_ = Timestamp::Unset(); } inline Packet& Packet::operator=(Packet&& packet) { - VLOG(2) << "Using move assignment operator of " << packet.DebugString(); + VLOG(4) << "Using move assignment operator of " << packet.DebugString(); if (this != &packet) { holder_ = std::move(packet.holder_); timestamp_ = packet.timestamp_; diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index 2a7a87e11..fcef2af94 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -246,6 +246,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":opencv_core", + "//mediapipe/framework:port", "//third_party:opencv", ], ) diff --git a/mediapipe/framework/port/build_config.bzl b/mediapipe/framework/port/build_config.bzl index 4699f10c4..ccb074977 100644 --- a/mediapipe/framework/port/build_config.bzl +++ b/mediapipe/framework/port/build_config.bzl @@ -5,6 +5,98 @@ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") +def provided_args(**kwargs): + """Returns the keyword arguments omitting None arguments.""" + return {k: v for k, v in kwargs.items() if v != None} + +def allowed_args(**kwargs): + """Returns the keyword arguments allowed for proto_library(). + + Args: + **kwargs: the specified keyword arguments. + Returns: + the allowed keyword arguments. + """ + result = dict(kwargs) + result.pop("cc_api_version", None) + return result + +# TODO: load this macro from a common helper file. +def mediapipe_proto_library( + name, + srcs, + deps = [], + visibility = None, + testonly = 0, + compatible_with = [], + def_proto = True, + def_cc_proto = True, + def_py_proto = True, + def_java_lite_proto = True, + def_portable_proto = True, + portable_deps = None): + """Defines the proto_library targets needed for all mediapipe platforms. + + Args: + name: the new proto_library target name. + srcs: the ".proto" source files to compile. + deps: the proto_library targets for all referenced protobufs. + portable_deps: the portable_proto_library targets for all referenced protobufs. + visibility: visibility of this target. + testonly: true means the proto can be used for testing only. + compatible_with: see go/target-constraints. + def_proto: define the proto_library target + def_cc_proto: define the cc_proto_library target + def_py_proto: define the py_proto_library target + def_java_lite_proto: define the java_lite_proto_library target + def_portable_proto: define the portable_proto_library target + """ + _ignore = [def_portable_proto, portable_deps] + + # The proto_library targets for the compiled ".proto" source files. + proto_deps = [":" + name] + + if def_proto: + native.proto_library(**allowed_args(**provided_args( + name = name, + srcs = srcs, + deps = deps, + visibility = visibility, + testonly = testonly, + cc_api_version = 2, + compatible_with = compatible_with, + ))) + + if def_cc_proto: + cc_deps = [dep.replace("_proto", "_cc_proto") for dep in deps] + mediapipe_cc_proto_library(**provided_args( + name = name.replace("_proto", "_cc_proto"), + srcs = srcs, + deps = proto_deps, + cc_deps = cc_deps, + visibility = visibility, + testonly = testonly, + )) + + if def_py_proto: + py_deps = [dep.replace("_proto", "_py_pb2") for dep in deps] + mediapipe_py_proto_library(**provided_args( + name = name.replace("_proto", "_py_pb2"), + srcs = srcs, + proto_deps = proto_deps, + py_proto_deps = py_deps, + visibility = visibility, + api_version = 2, + )) + + if def_java_lite_proto: + native.java_lite_proto_library(**provided_args( + name = name.replace("_proto", "_java_proto_lite"), + deps = proto_deps, + strict_deps = 0, + visibility = visibility, + )) + def mediapipe_py_proto_library( name, srcs, @@ -21,14 +113,14 @@ def mediapipe_py_proto_library( py_proto_deps: a list of dependency labels for Bazel use; must be py_proto_library. """ _ignore = [api_version, proto_deps] - py_proto_library( + py_proto_library(**provided_args( name = name, srcs = srcs, visibility = visibility, default_runtime = "@com_google_protobuf//:protobuf_python", protoc = "@com_google_protobuf//:protoc", deps = py_proto_deps + ["@com_google_protobuf//:protobuf_python"], - ) + )) def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [], testonly = 0): """Generate cc_proto_library for mediapipe open source version. @@ -41,7 +133,7 @@ def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [], testonly: test only proto or not. """ _ignore = [deps] - cc_proto_library( + cc_proto_library(**provided_args( name = name, srcs = srcs, visibility = visibility, @@ -51,4 +143,4 @@ def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [], protoc = "@com_google_protobuf//:protoc", default_runtime = "@com_google_protobuf//:protobuf", alwayslink = 1, - ) + )) diff --git a/mediapipe/framework/port/opencv_video_inc.h b/mediapipe/framework/port/opencv_video_inc.h index 80b2c6251..dc84bf59b 100644 --- a/mediapipe/framework/port/opencv_video_inc.h +++ b/mediapipe/framework/port/opencv_video_inc.h @@ -17,6 +17,7 @@ #include +#include "mediapipe/framework/port.h" #include "mediapipe/framework/port/opencv_core_inc.h" #ifdef CV_VERSION_EPOCH // for OpenCV 2.x @@ -83,7 +84,7 @@ inline int fourcc(char c1, char c2, char c3, char c4) { #include #include -#if CV_VERSION_MAJOR == 4 +#if CV_VERSION_MAJOR == 4 && !defined(MEDIAPIPE_MOBILE) #include namespace cv { diff --git a/mediapipe/framework/profiler/BUILD b/mediapipe/framework/profiler/BUILD index 82a8bf123..f3d0d0ddc 100644 --- a/mediapipe/framework/profiler/BUILD +++ b/mediapipe/framework/profiler/BUILD @@ -270,17 +270,15 @@ cc_library( srcs = select({ "//conditions:default": ["profiler_resource_util.cc"], "//mediapipe:android": ["profiler_resource_util_android.cc"], - "//mediapipe:apple": ["profiler_resource_util_apple.cc"], - "//mediapipe:macos": ["profiler_resource_util.cc"], + "//mediapipe:ios": ["profiler_resource_util_ios.cc"], }), hdrs = ["profiler_resource_util.h"], # We use Objective-C++ on iOS. copts = select({ "//conditions:default": [], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "-ObjC++", ], - "//mediapipe:macos": [], }), visibility = [ "//mediapipe/framework:mediapipe_internal", diff --git a/mediapipe/framework/profiler/graph_profiler_stub.h b/mediapipe/framework/profiler/graph_profiler_stub.h index eca1b0dba..a21ca0ed2 100644 --- a/mediapipe/framework/profiler/graph_profiler_stub.h +++ b/mediapipe/framework/profiler/graph_profiler_stub.h @@ -53,6 +53,8 @@ class TraceEvent { GPU_TASK, DSP_TASK, TPU_TASK, + GPU_CALIBRATION, + PACKET_QUEUED, }; TraceEvent(const EventType& event_type) {} TraceEvent() {} @@ -69,6 +71,7 @@ class TraceEvent { inline TraceEvent& set_packet_data_id(const Packet* packet) { return *this; } inline TraceEvent& set_thread_id(int thread_id) { return *this; } inline TraceEvent& set_is_finish(bool is_finish) { return *this; } + inline TraceEvent& set_event_data(int64 data) { return *this; } }; // Empty implementation of ProfilingContext to be used in place of the diff --git a/mediapipe/framework/profiler/graph_tracer.cc b/mediapipe/framework/profiler/graph_tracer.cc index 0f599879f..eeadb6c76 100644 --- a/mediapipe/framework/profiler/graph_tracer.cc +++ b/mediapipe/framework/profiler/graph_tracer.cc @@ -26,6 +26,7 @@ namespace mediapipe { namespace { +using EventType = GraphTrace::EventType; const absl::Duration kDefaultTraceLogInterval = absl::Milliseconds(500); @@ -52,14 +53,18 @@ int64 GraphTracer::GetTraceLogCapacity() { GraphTracer::GraphTracer(const ProfilerConfig& profiler_config) : profiler_config_(profiler_config), trace_buffer_(GetTraceLogCapacity()) { - event_types_disabled_.resize(static_cast(GraphTrace::EventType_MAX + 1)); - for (int32 event_type : profiler_config_.trace_event_types_disabled()) { - event_types_disabled_[event_type] = true; + for (int disabled : profiler_config_.trace_event_types_disabled()) { + EventType event_type = static_cast(disabled); + (*trace_event_registry())[event_type].set_enabled(false); } } +TraceEventRegistry* GraphTracer::trace_event_registry() { + return trace_builder_.trace_event_registry(); +} + void GraphTracer::LogEvent(TraceEvent event) { - if (event_types_disabled_[static_cast(event.event_type)]) { + if (!(*trace_event_registry())[event.event_type].enabled()) { return; } event.set_thread_id(GetCurrentThreadId()); diff --git a/mediapipe/framework/profiler/graph_tracer.h b/mediapipe/framework/profiler/graph_tracer.h index 16cae77d5..89b43b58b 100644 --- a/mediapipe/framework/profiler/graph_tracer.h +++ b/mediapipe/framework/profiler/graph_tracer.h @@ -55,6 +55,9 @@ class GraphTracer { // Create a tracer to record up to |capacity| recent events. GraphTracer(const ProfilerConfig& profiler_config); + // Returns the registry of trace event types. + TraceEventRegistry* trace_event_registry(); + // Append a TraceEvent to the TraceBuffer. void LogEvent(TraceEvent event); @@ -85,9 +88,6 @@ class GraphTracer { // The settings for this tracer. ProfilerConfig profiler_config_; - // Indicates event types that will not be logged. - std::vector event_types_disabled_; - // The circular buffer of TraceEvents. TraceBuffer trace_buffer_; diff --git a/mediapipe/framework/profiler/graph_tracer_test.cc b/mediapipe/framework/profiler/graph_tracer_test.cc index e34ebb807..70ee39873 100644 --- a/mediapipe/framework/profiler/graph_tracer_test.cc +++ b/mediapipe/framework/profiler/graph_tracer_test.cc @@ -161,7 +161,7 @@ TEST_F(GraphTracerTest, CalculatorTrace) { finish_time: 0 packet_timestamp: 0 stream_id: 1 - packet_id: 1 + event_data: 1 } output_trace { packet_timestamp: 0 stream_id: 2 } } @@ -236,7 +236,7 @@ TEST_F(GraphTracerTest, GraphTrace) { finish_time: 0 packet_timestamp: 0 stream_id: 1 - packet_id: 1 + event_data: 1 } output_trace { packet_timestamp: 0 stream_id: 2 } output_trace { packet_timestamp: 0 stream_id: 3 } @@ -254,7 +254,7 @@ TEST_F(GraphTracerTest, GraphTrace) { finish_time: 11000 packet_timestamp: 0 stream_id: 2 - packet_id: 2 + event_data: 2 } output_trace { packet_timestamp: 0 stream_id: 4 } } @@ -270,7 +270,7 @@ TEST_F(GraphTracerTest, GraphTrace) { finish_time: 16000 packet_timestamp: 0 stream_id: 3 - packet_id: 3 + event_data: 3 } output_trace { packet_timestamp: 0 stream_id: 5 } } @@ -286,7 +286,7 @@ TEST_F(GraphTracerTest, GraphTrace) { finish_time: 38000 packet_timestamp: 5 stream_id: 3 - packet_id: 4 + event_data: 4 } output_trace { packet_timestamp: 5 stream_id: 5 } } @@ -452,12 +452,15 @@ class GraphTracerE2ETest : public ::testing::Test { } } void StripDataIds(GraphTrace* trace) { + TraceBuilder builder; for (auto& ct : *trace->mutable_calculator_trace()) { - for (auto& st : *ct.mutable_input_trace()) { - st.clear_packet_id(); - } - for (auto& st : *ct.mutable_output_trace()) { - st.clear_packet_id(); + if ((*builder.trace_event_registry())[ct.event_type()].id_event_data()) { + for (auto& st : *ct.mutable_input_trace()) { + st.clear_event_data(); + } + for (auto& st : *ct.mutable_output_trace()) { + st.clear_event_data(); + } } } } @@ -656,63 +659,81 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) { calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } + calculator_trace { node_id: 1 input_timestamp: 10000 } + calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } + calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 0 input_timestamp: 20000 } + calculator_trace { node_id: 1 input_timestamp: 20000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 20000 } calculator_trace { node_id: 1 input_timestamp: 20000 } + calculator_trace { node_id: 2 input_timestamp: 20000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 20000 } calculator_trace { node_id: 2 input_timestamp: 20000 } + calculator_trace { node_id: 4 input_timestamp: 20000 } calculator_trace { node_id: 4 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 20000 } calculator_trace { node_id: 0 input_timestamp: 30000 } + calculator_trace { node_id: 1 input_timestamp: 30000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 30000 } calculator_trace { node_id: 1 input_timestamp: 30000 } + calculator_trace { node_id: 2 input_timestamp: 30000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 30000 } calculator_trace { node_id: 2 input_timestamp: 30000 } + calculator_trace { node_id: 3 input_timestamp: 30000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 0 input_timestamp: 40000 } + calculator_trace { node_id: 1 input_timestamp: 40000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 40000 } calculator_trace { node_id: 1 input_timestamp: 40000 } + calculator_trace { node_id: 2 input_timestamp: 40000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 40000 } calculator_trace { node_id: 2 input_timestamp: 40000 } + calculator_trace { node_id: 4 input_timestamp: 40000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } + calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } + calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 30000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 0 input_timestamp: 50000 } + calculator_trace { node_id: 1 input_timestamp: 50000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 50000 } calculator_trace { node_id: 1 input_timestamp: 50000 } + calculator_trace { node_id: 2 input_timestamp: 50000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 2 input_timestamp: 50000 } calculator_trace { node_id: 2 input_timestamp: 50000 } + calculator_trace { node_id: 3 input_timestamp: 50000 } calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 0 input_timestamp: 60000 } + calculator_trace { node_id: 1 input_timestamp: 60000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 60000 } calculator_trace { node_id: 2 input_timestamp: 10000 } @@ -721,33 +742,39 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) { calculator_trace { node_id: 2 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 20000 } + calculator_trace { node_id: 5 input_timestamp: 20000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 20000 } calculator_trace { node_id: 5 input_timestamp: 20000 } calculator_trace { node_id: 5 input_timestamp: 20000 } + calculator_trace { node_id: 1 input_timestamp: 20000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 40000 } calculator_trace { node_id: 1 input_timestamp: 20000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 30000 } + calculator_trace { node_id: 5 input_timestamp: 30000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 30000 } calculator_trace { node_id: 5 input_timestamp: 30000 } calculator_trace { node_id: 5 input_timestamp: 30000 } + calculator_trace { node_id: 1 input_timestamp: 30000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 50000 } calculator_trace { node_id: 1 input_timestamp: 30000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 50000 } + calculator_trace { node_id: 5 input_timestamp: 50000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 3 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 50000 } calculator_trace { node_id: 5 input_timestamp: 50000 } calculator_trace { node_id: 5 input_timestamp: 50000 } + calculator_trace { node_id: 1 input_timestamp: 50000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } @@ -755,16 +782,17 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) { calculator_trace { node_id: 1 input_timestamp: 50000 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 40000 } + calculator_trace { node_id: 5 input_timestamp: 40000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 4 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 40000 } calculator_trace { node_id: 5 input_timestamp: 40000 } + calculator_trace { node_id: 1 input_timestamp: 50001 } calculator_trace { node_id: 1 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 5 input_timestamp: 10000 } calculator_trace { node_id: 1 input_timestamp: 50001 } - calculator_trace { node_id: 1 input_timestamp: 10000 } - )"))); + calculator_trace { node_id: 1 input_timestamp: 10000 })"))); // Validate a one-timestamp slice of the event trace. GraphTrace trace_2; @@ -773,142 +801,179 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) { &trace_2); StripThreadIds(&trace_2); StripDataIds(&trace_2); - EXPECT_THAT(trace_2, - EqualsProto(::mediapipe::ParseTextProtoOrDie( - R"( - base_time: 1544086800000000 - base_timestamp: 10000 - stream_name: "" - stream_name: "input_packets_0" - stream_name: "input_0_sampled" - stream_name: "input_0" - stream_name: "input_1" - stream_name: "output_0" - stream_name: "output_packets_0" - stream_name: "finish_indicator" - stream_name: "output_1" - calculator_trace { - node_id: 3 - input_timestamp: 0 - event_type: PROCESS - finish_time: 25002 - output_trace { packet_timestamp: 0 stream_id: 5 } - } - calculator_trace { - node_id: 5 - event_type: READY_FOR_PROCESS - start_time: 25002 - } - calculator_trace { - node_id: 3 - event_type: READY_FOR_PROCESS - start_time: 25002 - } - calculator_trace { - node_id: 5 - input_timestamp: 0 - event_type: PROCESS - start_time: 25002 - input_trace { packet_timestamp: 0 stream_id: 5 } - } - calculator_trace { - node_id: 5 - input_timestamp: 0 - event_type: PROCESS - finish_time: 25002 - output_trace { packet_timestamp: 0 stream_id: 6 } - } - calculator_trace { - node_id: 5 - input_timestamp: 0 - event_type: PROCESS - finish_time: 25002 - output_trace { packet_timestamp: 0 stream_id: 7 } - } - calculator_trace { - node_id: 1 - event_type: READY_FOR_PROCESS - start_time: 25002 - } - calculator_trace { - node_id: 5 - event_type: NOT_READY - start_time: 25002 - } - calculator_trace { - node_id: 3 - input_timestamp: 20000 - event_type: PROCESS - start_time: 25002 - input_trace { packet_timestamp: 20000 stream_id: 3 } - } - calculator_trace { - node_id: 1 - input_timestamp: 0 - event_type: PROCESS - start_time: 25002 - input_trace { packet_timestamp: 0 stream_id: 7 } - } - calculator_trace { - node_id: 1 - event_type: NOT_READY - start_time: 25002 - } - calculator_trace { - node_id: 0 - input_timestamp: 40000 - event_type: PROCESS - finish_time: 25005 - output_trace { packet_timestamp: 40000 stream_id: 1 } - } - calculator_trace { - node_id: 1 - event_type: READY_FOR_PROCESS - start_time: 25005 - } - calculator_trace { - node_id: 1 - input_timestamp: 40000 - event_type: PROCESS - start_time: 25005 - input_trace { packet_timestamp: 40000 stream_id: 1 } - } - calculator_trace { - node_id: 1 - input_timestamp: 40000 - event_type: PROCESS - finish_time: 25005 - output_trace { packet_timestamp: 40000 stream_id: 2 } - } - calculator_trace { - node_id: 2 - event_type: READY_FOR_PROCESS - start_time: 25005 - } - calculator_trace { - node_id: 1 - event_type: NOT_READY - start_time: 25005 - } - calculator_trace { - node_id: 2 - input_timestamp: 40000 - event_type: PROCESS - start_time: 25005 - input_trace { packet_timestamp: 40000 stream_id: 2 } - } - calculator_trace { - node_id: 2 - input_timestamp: 40000 - event_type: PROCESS - finish_time: 25005 - output_trace { packet_timestamp: 40000 stream_id: 3 } - } - calculator_trace { - node_id: 2 - event_type: NOT_READY - start_time: 25005 - })"))); + EXPECT_THAT( + trace_2, + EqualsProto(::mediapipe::ParseTextProtoOrDie( + R"( + base_time: 1544086800000000 + base_timestamp: 10000 + stream_name: "" + stream_name: "input_packets_0" + stream_name: "input_0_sampled" + stream_name: "input_0" + stream_name: "input_1" + stream_name: "output_0" + stream_name: "output_packets_0" + stream_name: "finish_indicator" + stream_name: "output_1" + calculator_trace { + node_id: 3 + input_timestamp: 0 + event_type: PROCESS + finish_time: 25002 + output_trace { packet_timestamp: 0 stream_id: 5 } + } + calculator_trace { + node_id: 5 + input_timestamp: 0 + event_type: PACKET_QUEUED + start_time: 25002 + input_trace { packet_timestamp: 0 stream_id: 5 event_data: 1 } + } + calculator_trace { + node_id: 5 + event_type: READY_FOR_PROCESS + start_time: 25002 + } + calculator_trace { + node_id: 3 + event_type: READY_FOR_PROCESS + start_time: 25002 + } + calculator_trace { + node_id: 5 + input_timestamp: 0 + event_type: PROCESS + start_time: 25002 + input_trace { packet_timestamp: 0 stream_id: 5 } + } + calculator_trace { + node_id: 5 + input_timestamp: 0 + event_type: PROCESS + finish_time: 25002 + output_trace { packet_timestamp: 0 stream_id: 6 } + } + calculator_trace { + node_id: 5 + input_timestamp: 0 + event_type: PROCESS + finish_time: 25002 + output_trace { packet_timestamp: 0 stream_id: 7 } + } + calculator_trace { + node_id: 1 + input_timestamp: 0 + event_type: PACKET_QUEUED + start_time: 25002 + input_trace { packet_timestamp: 0 stream_id: 7 event_data: 1 } + } + calculator_trace { + node_id: 1 + event_type: READY_FOR_PROCESS + start_time: 25002 + } + calculator_trace { + node_id: 5 + event_type: NOT_READY + start_time: 25002 + } + calculator_trace { + node_id: 3 + input_timestamp: 20000 + event_type: PROCESS + start_time: 25002 + input_trace { packet_timestamp: 20000 stream_id: 3 } + } + calculator_trace { + node_id: 1 + input_timestamp: 0 + event_type: PROCESS + start_time: 25002 + input_trace { packet_timestamp: 0 stream_id: 7 } + } + calculator_trace { + node_id: 1 + event_type: NOT_READY + start_time: 25002 + } + calculator_trace { + node_id: 0 + input_timestamp: 40000 + event_type: PROCESS + finish_time: 25005 + output_trace { packet_timestamp: 40000 stream_id: 1 } + } + calculator_trace { + node_id: 1 + input_timestamp: 40000 + event_type: PACKET_QUEUED + start_time: 25005 + input_trace { packet_timestamp: 40000 stream_id: 1 event_data: 1 } + } + calculator_trace { + node_id: 1 + event_type: READY_FOR_PROCESS + start_time: 25005 + } + calculator_trace { + node_id: 1 + input_timestamp: 40000 + event_type: PROCESS + start_time: 25005 + input_trace { packet_timestamp: 40000 stream_id: 1 } + } + calculator_trace { + node_id: 1 + input_timestamp: 40000 + event_type: PROCESS + finish_time: 25005 + output_trace { packet_timestamp: 40000 stream_id: 2 } + } + calculator_trace { + node_id: 2 + input_timestamp: 40000 + event_type: PACKET_QUEUED + start_time: 25005 + input_trace { packet_timestamp: 40000 stream_id: 2 event_data: 1 } + } + calculator_trace { + node_id: 2 + event_type: READY_FOR_PROCESS + start_time: 25005 + } + calculator_trace { + node_id: 1 + event_type: NOT_READY + start_time: 25005 + } + calculator_trace { + node_id: 2 + input_timestamp: 40000 + event_type: PROCESS + start_time: 25005 + input_trace { packet_timestamp: 40000 stream_id: 2 } + } + calculator_trace { + node_id: 2 + input_timestamp: 40000 + event_type: PROCESS + finish_time: 25005 + output_trace { packet_timestamp: 40000 stream_id: 3 } + } + calculator_trace { + node_id: 3 + input_timestamp: 40000 + event_type: PACKET_QUEUED + start_time: 25005 + input_trace { packet_timestamp: 40000 stream_id: 3 event_data: 1 } + } + calculator_trace { + node_id: 2 + event_type: NOT_READY + start_time: 25005 + } + )"))); } // Read a GraphProfile from a file path. @@ -931,7 +996,7 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFile) { GraphProfile profile; MP_EXPECT_OK( ReadGraphProfile(absl::StrCat(log_path, 0, ".binarypb"), &profile)); - EXPECT_EQ(89, profile.graph_trace(0).calculator_trace().size()); + EXPECT_EQ(111, profile.graph_trace(0).calculator_trace().size()); } TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) { @@ -956,7 +1021,11 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) { graph_profiles.push_back(profile); } } - std::vector expected = {37, 52, 9}; + + // The expected counts of calculator_trace records in each of the log files. + // The processing spans three 12.5ms log files, because + // RunDemuxInFlightGraph adds packets over 30ms. + std::vector expected = {49, 64, 11}; EXPECT_EQ(event_counts, expected); GraphProfile& profile_2 = graph_profiles[2]; profile_2.clear_calculator_profiles(); @@ -992,6 +1061,18 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) { finish_time: 70004 output_trace { packet_timestamp: 40000 stream_id: 8 } } + calculator_trace { + node_id: 5 + input_timestamp: 40000 + event_type: PACKET_QUEUED + start_time: 70004 + input_trace { + finish_time: 70004 + packet_timestamp: 40000 + stream_id: 8 + event_data: 1 + } + } calculator_trace { node_id: 5 event_type: READY_FOR_PROCESS @@ -1016,6 +1097,18 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) { } output_trace { packet_timestamp: 50001 stream_id: 7 } } + calculator_trace { + node_id: 1 + input_timestamp: 50001 + event_type: PACKET_QUEUED + start_time: 70004 + input_trace { + finish_time: 70004 + packet_timestamp: 50001 + stream_id: 7 + event_data: 1 + } + } calculator_trace { node_id: 1 event_type: READY_FOR_PROCESS @@ -1199,7 +1292,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) { finish_time: 0 packet_timestamp: 0 stream_id: 1 - packet_id: 0 + event_data: 0 } output_trace { packet_timestamp: 0 stream_id: 2 } thread_id: 0 @@ -1231,7 +1324,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) { input_timestamp: 0 event_type: PROCESS start_time: 0 - input_trace { packet_timestamp: 0 stream_id: 1 packet_id: 0 } + input_trace { packet_timestamp: 0 stream_id: 1 event_data: 0 } thread_id: 0 } calculator_trace { @@ -1253,7 +1346,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) { input_timestamp: 0 event_type: PROCESS finish_time: 1000 - output_trace { packet_timestamp: 0 stream_id: 2 packet_id: 0 } + output_trace { packet_timestamp: 0 stream_id: 2 event_data: 0 } thread_id: 0 } )"))); diff --git a/mediapipe/framework/profiler/profiler_resource_util_apple.cc b/mediapipe/framework/profiler/profiler_resource_util_ios.cc similarity index 100% rename from mediapipe/framework/profiler/profiler_resource_util_apple.cc rename to mediapipe/framework/profiler/profiler_resource_util_ios.cc diff --git a/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt b/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt index 9d5e30426..2a4c36a37 100644 --- a/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt +++ b/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt @@ -54,7 +54,7 @@ graph_trace: { finish_time : 57788 packet_timestamp: 0 stream_id : 2 - packet_id : 1 + event_data : 1 } output_trace: { packet_timestamp: 0 @@ -83,7 +83,7 @@ graph_trace: { finish_time : 73331 packet_timestamp: -9223372036854775806 stream_id : 1 - packet_id : 2 + event_data : 2 } thread_id : 1 } @@ -115,7 +115,7 @@ graph_trace: { finish_time : 75586 packet_timestamp: 33333 stream_id : 2 - packet_id : 3 + event_data : 3 } output_trace: { packet_timestamp: 33333 @@ -157,7 +157,7 @@ graph_trace: { finish_time : 102231 packet_timestamp: 66666 stream_id : 2 - packet_id : 4 + event_data : 4 } output_trace: { packet_timestamp: 66666 @@ -199,7 +199,7 @@ graph_trace: { finish_time : 130097 packet_timestamp: 100000 stream_id : 2 - packet_id : 5 + event_data : 5 } output_trace: { packet_timestamp: 100000 @@ -241,7 +241,7 @@ graph_trace: { finish_time : 158296 packet_timestamp: 133333 stream_id : 2 - packet_id : 6 + event_data : 6 } output_trace: { packet_timestamp: 133333 @@ -271,7 +271,7 @@ graph_trace: { finish_time : 178974 packet_timestamp: 0 stream_id : 3 - packet_id : 7 + event_data : 7 } thread_id : 1 } @@ -303,7 +303,7 @@ graph_trace: { finish_time : 186770 packet_timestamp: 166666 stream_id : 2 - packet_id : 3 + event_data : 3 } output_trace: { packet_timestamp: 166666 @@ -345,7 +345,7 @@ graph_trace: { finish_time : 212094 packet_timestamp: 200000 stream_id : 2 - packet_id : 8 + event_data : 8 } output_trace: { packet_timestamp: 200000 @@ -369,7 +369,7 @@ graph_trace: { finish_time : 224921 packet_timestamp: 33333 stream_id : 3 - packet_id : 9 + event_data : 9 } thread_id : 1 } @@ -407,7 +407,7 @@ graph_trace: { finish_time : 238877 packet_timestamp: 233333 stream_id : 2 - packet_id : 3 + event_data : 3 } output_trace: { packet_timestamp: 233333 @@ -431,7 +431,7 @@ graph_trace: { finish_time : 244835 packet_timestamp: 66666 stream_id : 3 - packet_id : 10 + event_data : 10 } thread_id : 1 } @@ -469,7 +469,7 @@ graph_trace: { finish_time : 264756 packet_timestamp: 266666 stream_id : 2 - packet_id : 9 + event_data : 9 } output_trace: { packet_timestamp: 266666 @@ -493,7 +493,7 @@ graph_trace: { finish_time : 269005 packet_timestamp: 100000 stream_id : 3 - packet_id : 11 + event_data : 11 } thread_id : 1 } @@ -531,7 +531,7 @@ graph_trace: { finish_time : 288787 packet_timestamp: 300000 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 300000 @@ -555,7 +555,7 @@ graph_trace: { finish_time : 297923 packet_timestamp: 133333 stream_id : 3 - packet_id : 12 + event_data : 12 } thread_id : 1 } @@ -581,7 +581,7 @@ graph_trace: { finish_time : 315500 packet_timestamp: 166666 stream_id : 3 - packet_id : 13 + event_data : 13 } thread_id : 1 } @@ -613,7 +613,7 @@ graph_trace: { finish_time : 319060 packet_timestamp: 333333 stream_id : 2 - packet_id : 11 + event_data : 11 } output_trace: { packet_timestamp: 333333 @@ -655,7 +655,7 @@ graph_trace: { finish_time : 343740 packet_timestamp: 366666 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 366666 @@ -685,7 +685,7 @@ graph_trace: { finish_time : 358594 packet_timestamp: 200000 stream_id : 3 - packet_id : 1 + event_data : 1 } thread_id : 1 } @@ -717,7 +717,7 @@ graph_trace: { finish_time : 371485 packet_timestamp: 400000 stream_id : 2 - packet_id : 15 + event_data : 15 } output_trace: { packet_timestamp: 400000 @@ -741,7 +741,7 @@ graph_trace: { finish_time : 381603 packet_timestamp: 233333 stream_id : 3 - packet_id : 16 + event_data : 16 } thread_id : 1 } @@ -784,7 +784,7 @@ graph_trace: { finish_time : 398472 packet_timestamp: 433333 stream_id : 2 - packet_id : 17 + event_data : 17 } output_trace: { packet_timestamp: 433333 @@ -807,7 +807,7 @@ graph_trace: { finish_time : 405325 packet_timestamp: 266666 stream_id : 3 - packet_id : 18 + event_data : 18 } thread_id : 1 } @@ -845,7 +845,7 @@ graph_trace: { finish_time : 424091 packet_timestamp: 466666 stream_id : 2 - packet_id : 16 + event_data : 16 } output_trace: { packet_timestamp: 466666 @@ -868,7 +868,7 @@ graph_trace: { finish_time : 425203 packet_timestamp: 300000 stream_id : 3 - packet_id : 19 + event_data : 19 } thread_id : 1 } @@ -906,7 +906,7 @@ graph_trace: { finish_time : 449157 packet_timestamp: 500000 stream_id : 2 - packet_id : 18 + event_data : 18 } output_trace: { packet_timestamp: 500000 @@ -929,7 +929,7 @@ graph_trace: { finish_time : 458501 packet_timestamp: 333333 stream_id : 3 - packet_id : 20 + event_data : 20 } thread_id : 1 } @@ -967,7 +967,7 @@ graph_trace: { finish_time : 475500 packet_timestamp: 533333 stream_id : 2 - packet_id : 21 + event_data : 21 } output_trace: { packet_timestamp: 533333 @@ -990,7 +990,7 @@ graph_trace: { finish_time : 477113 packet_timestamp: 366666 stream_id : 3 - packet_id : 5 + event_data : 5 } thread_id : 1 } @@ -1028,7 +1028,7 @@ graph_trace: { finish_time : 500205 packet_timestamp: 566666 stream_id : 2 - packet_id : 20 + event_data : 20 } output_trace: { packet_timestamp: 566666 @@ -1051,7 +1051,7 @@ graph_trace: { finish_time : 506708 packet_timestamp: 400000 stream_id : 3 - packet_id : 14 + event_data : 14 } thread_id : 1 } @@ -1089,7 +1089,7 @@ graph_trace: { finish_time : 524099 packet_timestamp: 600000 stream_id : 2 - packet_id : 5 + event_data : 5 } output_trace: { packet_timestamp: 600000 @@ -1113,7 +1113,7 @@ graph_trace: { finish_time : 527109 packet_timestamp: 433333 stream_id : 3 - packet_id : 1 + event_data : 1 } thread_id : 1 } @@ -1151,7 +1151,7 @@ graph_trace: { finish_time : 549237 packet_timestamp: 633333 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 633333 @@ -1181,7 +1181,7 @@ graph_trace: { finish_time : 563962 packet_timestamp: 466666 stream_id : 3 - packet_id : 10 + event_data : 10 } thread_id : 1 } @@ -1213,7 +1213,7 @@ graph_trace: { finish_time : 579872 packet_timestamp: 666666 stream_id : 2 - packet_id : 5 + event_data : 5 } output_trace: { packet_timestamp: 666666 @@ -1237,7 +1237,7 @@ graph_trace: { finish_time : 587948 packet_timestamp: 500000 stream_id : 3 - packet_id : 22 + event_data : 22 } thread_id : 1 } @@ -1275,7 +1275,7 @@ graph_trace: { finish_time : 602586 packet_timestamp: 700000 stream_id : 2 - packet_id : 23 + event_data : 23 } output_trace: { packet_timestamp: 700000 @@ -1299,7 +1299,7 @@ graph_trace: { finish_time : 607961 packet_timestamp: 533333 stream_id : 3 - packet_id : 19 + event_data : 19 } thread_id : 1 } @@ -1325,7 +1325,7 @@ graph_trace: { finish_time : 625447 packet_timestamp: 566666 stream_id : 3 - packet_id : 17 + event_data : 17 } thread_id : 1 } @@ -1357,7 +1357,7 @@ graph_trace: { finish_time : 627152 packet_timestamp: 733333 stream_id : 2 - packet_id : 22 + event_data : 22 } output_trace: { packet_timestamp: 733333 @@ -1381,7 +1381,7 @@ graph_trace: { finish_time : 643316 packet_timestamp: 600000 stream_id : 3 - packet_id : 24 + event_data : 24 } thread_id : 1 } @@ -1419,7 +1419,7 @@ graph_trace: { finish_time : 659152 packet_timestamp: 766666 stream_id : 2 - packet_id : 17 + event_data : 17 } output_trace: { packet_timestamp: 766666 @@ -1443,7 +1443,7 @@ graph_trace: { finish_time : 663938 packet_timestamp: 633333 stream_id : 3 - packet_id : 7 + event_data : 7 } thread_id : 1 } @@ -1481,7 +1481,7 @@ graph_trace: { finish_time : 683058 packet_timestamp: 800000 stream_id : 2 - packet_id : 24 + event_data : 24 } output_trace: { packet_timestamp: 800000 @@ -1505,7 +1505,7 @@ graph_trace: { finish_time : 683231 packet_timestamp: 666666 stream_id : 3 - packet_id : 25 + event_data : 25 } thread_id : 1 } @@ -1543,7 +1543,7 @@ graph_trace: { finish_time : 706942 packet_timestamp: 833333 stream_id : 2 - packet_id : 7 + event_data : 7 } output_trace: { packet_timestamp: 833333 @@ -1567,7 +1567,7 @@ graph_trace: { finish_time : 715525 packet_timestamp: 700000 stream_id : 3 - packet_id : 26 + event_data : 26 } thread_id : 1 } @@ -1605,7 +1605,7 @@ graph_trace: { finish_time : 732560 packet_timestamp: 866666 stream_id : 2 - packet_id : 24 + event_data : 24 } output_trace: { packet_timestamp: 866666 @@ -1629,7 +1629,7 @@ graph_trace: { finish_time : 733658 packet_timestamp: 733333 stream_id : 3 - packet_id : 27 + event_data : 27 } thread_id : 1 } @@ -1667,7 +1667,7 @@ graph_trace: { finish_time : 757339 packet_timestamp: 900000 stream_id : 2 - packet_id : 26 + event_data : 26 } output_trace: { packet_timestamp: 900000 @@ -1691,7 +1691,7 @@ graph_trace: { finish_time : 763007 packet_timestamp: 766666 stream_id : 3 - packet_id : 23 + event_data : 23 } thread_id : 1 } @@ -1729,7 +1729,7 @@ graph_trace: { finish_time : 780390 packet_timestamp: 933333 stream_id : 2 - packet_id : 27 + event_data : 27 } output_trace: { packet_timestamp: 933333 @@ -1753,7 +1753,7 @@ graph_trace: { finish_time : 782325 packet_timestamp: 800000 stream_id : 3 - packet_id : 17 + event_data : 17 } thread_id : 1 } @@ -1791,7 +1791,7 @@ graph_trace: { finish_time : 802623 packet_timestamp: 966666 stream_id : 2 - packet_id : 23 + event_data : 23 } output_trace: { packet_timestamp: 966666 @@ -1815,7 +1815,7 @@ graph_trace: { finish_time : 811275 packet_timestamp: 833333 stream_id : 3 - packet_id : 10 + event_data : 10 } thread_id : 1 } @@ -1853,7 +1853,7 @@ graph_trace: { finish_time : 829445 packet_timestamp: 1000000 stream_id : 2 - packet_id : 17 + event_data : 17 } output_trace: { packet_timestamp: 1000000 @@ -1877,7 +1877,7 @@ graph_trace: { finish_time : 829793 packet_timestamp: 866666 stream_id : 3 - packet_id : 28 + event_data : 28 } thread_id : 1 } @@ -1915,7 +1915,7 @@ graph_trace: { finish_time : 852994 packet_timestamp: 1033333 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 1033333 @@ -1939,7 +1939,7 @@ graph_trace: { finish_time : 859663 packet_timestamp: 900000 stream_id : 3 - packet_id : 29 + event_data : 29 } thread_id : 1 } @@ -1976,7 +1976,7 @@ graph_trace: { finish_time : 879819 packet_timestamp: 1066666 stream_id : 2 - packet_id : 28 + event_data : 28 } thread_id : 0 } @@ -1996,7 +1996,7 @@ graph_trace: { finish_time : 883409 packet_timestamp: 933333 stream_id : 3 - packet_id : 30 + event_data : 30 } thread_id : 1 } @@ -2050,7 +2050,7 @@ graph_trace: { finish_time : 905400 packet_timestamp: 1100000 stream_id : 2 - packet_id : 29 + event_data : 29 } output_trace: { packet_timestamp: 1100000 @@ -2079,7 +2079,7 @@ graph_trace: { finish_time : 920776 packet_timestamp: 966666 stream_id : 3 - packet_id : 31 + event_data : 31 } thread_id : 1 } @@ -2111,7 +2111,7 @@ graph_trace: { finish_time : 934268 packet_timestamp: 1133333 stream_id : 2 - packet_id : 32 + event_data : 32 } output_trace: { packet_timestamp: 1133333 @@ -2134,7 +2134,7 @@ graph_trace: { finish_time : 939597 packet_timestamp: 1000000 stream_id : 3 - packet_id : 23 + event_data : 23 } thread_id : 1 } @@ -2172,7 +2172,7 @@ graph_trace: { finish_time : 957408 packet_timestamp: 1166666 stream_id : 2 - packet_id : 31 + event_data : 31 } output_trace: { packet_timestamp: 1166666 @@ -2195,7 +2195,7 @@ graph_trace: { finish_time : 957881 packet_timestamp: 1033333 stream_id : 3 - packet_id : 33 + event_data : 33 } thread_id : 1 } @@ -2233,7 +2233,7 @@ graph_trace: { finish_time : 979949 packet_timestamp: 1200000 stream_id : 2 - packet_id : 23 + event_data : 23 } output_trace: { packet_timestamp: 1200000 @@ -2257,7 +2257,7 @@ graph_trace: { finish_time : 988502 packet_timestamp: 1066666 stream_id : 3 - packet_id : 34 + event_data : 34 } thread_id : 1 } @@ -2283,7 +2283,7 @@ graph_trace: { finish_time : 1007954 packet_timestamp: 1100000 stream_id : 3 - packet_id : 35 + event_data : 35 } thread_id : 1 } @@ -2315,7 +2315,7 @@ graph_trace: { finish_time : 1009962 packet_timestamp: 1233333 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 1233333 @@ -2339,7 +2339,7 @@ graph_trace: { finish_time : 1026043 packet_timestamp: 1133333 stream_id : 3 - packet_id : 29 + event_data : 29 } thread_id : 1 } @@ -2365,7 +2365,7 @@ graph_trace: { finish_time : 1043492 packet_timestamp: 1166666 stream_id : 3 - packet_id : 32 + event_data : 32 } thread_id : 1 } @@ -2397,7 +2397,7 @@ graph_trace: { finish_time : 1049243 packet_timestamp: 1266666 stream_id : 2 - packet_id : 35 + event_data : 35 } output_trace: { packet_timestamp: 1266666 @@ -2427,7 +2427,7 @@ graph_trace: { finish_time : 1073269 packet_timestamp: 1200000 stream_id : 3 - packet_id : 28 + event_data : 28 } thread_id : 1 } @@ -2459,7 +2459,7 @@ graph_trace: { finish_time : 1073830 packet_timestamp: 1300000 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 1300000 @@ -2489,7 +2489,7 @@ graph_trace: { finish_time : 1095743 packet_timestamp: 1233333 stream_id : 3 - packet_id : 37 + event_data : 37 } thread_id : 1 } @@ -2521,7 +2521,7 @@ graph_trace: { finish_time : 1109879 packet_timestamp: 1333333 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 1333333 @@ -2545,7 +2545,7 @@ graph_trace: { finish_time : 1114533 packet_timestamp: 1266666 stream_id : 3 - packet_id : 39 + event_data : 39 } thread_id : 1 } @@ -2583,7 +2583,7 @@ graph_trace: { finish_time : 1134249 packet_timestamp: 1366666 stream_id : 2 - packet_id : 37 + event_data : 37 } output_trace: { packet_timestamp: 1366666 @@ -2607,7 +2607,7 @@ graph_trace: { finish_time : 1134631 packet_timestamp: 1300000 stream_id : 3 - packet_id : 40 + event_data : 40 } thread_id : 1 } @@ -2645,7 +2645,7 @@ graph_trace: { finish_time : 1161869 packet_timestamp: 1400000 stream_id : 2 - packet_id : 39 + event_data : 39 } output_trace: { packet_timestamp: 1400000 @@ -2669,7 +2669,7 @@ graph_trace: { finish_time : 1170809 packet_timestamp: 1333333 stream_id : 3 - packet_id : 19 + event_data : 19 } thread_id : 1 } @@ -2707,7 +2707,7 @@ graph_trace: { finish_time : 1187129 packet_timestamp: 1433333 stream_id : 2 - packet_id : 37 + event_data : 37 } output_trace: { packet_timestamp: 1433333 @@ -2731,7 +2731,7 @@ graph_trace: { finish_time : 1190126 packet_timestamp: 1366666 stream_id : 3 - packet_id : 32 + event_data : 32 } thread_id : 1 } @@ -2769,7 +2769,7 @@ graph_trace: { finish_time : 1212856 packet_timestamp: 1466666 stream_id : 2 - packet_id : 19 + event_data : 19 } output_trace: { packet_timestamp: 1466666 @@ -2793,7 +2793,7 @@ graph_trace: { finish_time : 1217828 packet_timestamp: 1400000 stream_id : 3 - packet_id : 33 + event_data : 33 } thread_id : 1 } @@ -2819,7 +2819,7 @@ graph_trace: { finish_time : 1235898 packet_timestamp: 1433333 stream_id : 3 - packet_id : 41 + event_data : 41 } thread_id : 1 } @@ -2851,7 +2851,7 @@ graph_trace: { finish_time : 1246556 packet_timestamp: 1500000 stream_id : 2 - packet_id : 32 + event_data : 32 } output_trace: { packet_timestamp: 1500000 @@ -2881,7 +2881,7 @@ graph_trace: { finish_time : 1262154 packet_timestamp: 1466666 stream_id : 3 - packet_id : 42 + event_data : 42 } thread_id : 1 } @@ -2913,7 +2913,7 @@ graph_trace: { finish_time : 1277430 packet_timestamp: 1533333 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 1533333 @@ -2937,7 +2937,7 @@ graph_trace: { finish_time : 1283528 packet_timestamp: 1500000 stream_id : 3 - packet_id : 19 + event_data : 19 } thread_id : 1 } @@ -2975,7 +2975,7 @@ graph_trace: { finish_time : 1303444 packet_timestamp: 1566666 stream_id : 2 - packet_id : 42 + event_data : 42 } output_trace: { packet_timestamp: 1566666 @@ -3017,7 +3017,7 @@ graph_trace: { finish_time : 1329808 packet_timestamp: 1600000 stream_id : 2 - packet_id : 24 + event_data : 24 } output_trace: { packet_timestamp: 1600000 @@ -3059,7 +3059,7 @@ graph_trace: { finish_time : 1354971 packet_timestamp: 1633333 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 1633333 @@ -3100,7 +3100,7 @@ graph_trace: { finish_time : 1380340 packet_timestamp: 1666666 stream_id : 2 - packet_id : 43 + event_data : 43 } thread_id : 0 } @@ -3154,7 +3154,7 @@ graph_trace: { finish_time : 1407734 packet_timestamp: 1700000 stream_id : 2 - packet_id : 25 + event_data : 25 } output_trace: { packet_timestamp: 1700000 @@ -3196,7 +3196,7 @@ graph_trace: { finish_time : 1433471 packet_timestamp: 1733333 stream_id : 2 - packet_id : 40 + event_data : 40 } output_trace: { packet_timestamp: 1733333 @@ -3238,7 +3238,7 @@ graph_trace: { finish_time : 1458219 packet_timestamp: 1766666 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 1766666 @@ -3280,7 +3280,7 @@ graph_trace: { finish_time : 1484096 packet_timestamp: 1800000 stream_id : 2 - packet_id : 13 + event_data : 13 } output_trace: { packet_timestamp: 1800000 @@ -3322,7 +3322,7 @@ graph_trace: { finish_time : 1511535 packet_timestamp: 1833333 stream_id : 2 - packet_id : 44 + event_data : 44 } output_trace: { packet_timestamp: 1833333 @@ -3364,7 +3364,7 @@ graph_trace: { finish_time : 1536125 packet_timestamp: 1866666 stream_id : 2 - packet_id : 24 + event_data : 24 } output_trace: { packet_timestamp: 1866666 @@ -3406,7 +3406,7 @@ graph_trace: { finish_time : 1560223 packet_timestamp: 1900000 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 1900000 @@ -3448,7 +3448,7 @@ graph_trace: { finish_time : 1584387 packet_timestamp: 1933333 stream_id : 2 - packet_id : 13 + event_data : 13 } output_trace: { packet_timestamp: 1933333 @@ -3490,7 +3490,7 @@ graph_trace: { finish_time : 1609066 packet_timestamp: 1966666 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 1966666 @@ -3532,7 +3532,7 @@ graph_trace: { finish_time : 1633232 packet_timestamp: 2000000 stream_id : 2 - packet_id : 44 + event_data : 44 } output_trace: { packet_timestamp: 2000000 @@ -3574,7 +3574,7 @@ graph_trace: { finish_time : 1661755 packet_timestamp: 2033333 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 2033333 @@ -3616,7 +3616,7 @@ graph_trace: { finish_time : 1687803 packet_timestamp: 2066666 stream_id : 2 - packet_id : 13 + event_data : 13 } output_trace: { packet_timestamp: 2066666 @@ -3658,7 +3658,7 @@ graph_trace: { finish_time : 1713027 packet_timestamp: 2100000 stream_id : 2 - packet_id : 39 + event_data : 39 } output_trace: { packet_timestamp: 2100000 @@ -3700,7 +3700,7 @@ graph_trace: { finish_time : 1738094 packet_timestamp: 2133333 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 2133333 @@ -3742,7 +3742,7 @@ graph_trace: { finish_time : 1764014 packet_timestamp: 2166666 stream_id : 2 - packet_id : 46 + event_data : 46 } output_trace: { packet_timestamp: 2166666 @@ -3784,7 +3784,7 @@ graph_trace: { finish_time : 1789761 packet_timestamp: 2200000 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 2200000 @@ -3826,7 +3826,7 @@ graph_trace: { finish_time : 1816903 packet_timestamp: 2233333 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 2233333 @@ -3868,7 +3868,7 @@ graph_trace: { finish_time : 1842723 packet_timestamp: 2266666 stream_id : 2 - packet_id : 13 + event_data : 13 } output_trace: { packet_timestamp: 2266666 @@ -3910,7 +3910,7 @@ graph_trace: { finish_time : 1869293 packet_timestamp: 2300000 stream_id : 2 - packet_id : 47 + event_data : 47 } output_trace: { packet_timestamp: 2300000 @@ -3957,7 +3957,7 @@ graph_trace: { finish_time : 1896268 packet_timestamp: 2333333 stream_id : 2 - packet_id : 34 + event_data : 34 } output_trace: { packet_timestamp: 2333333 @@ -3999,7 +3999,7 @@ graph_trace: { finish_time : 1920552 packet_timestamp: 2366666 stream_id : 2 - packet_id : 29 + event_data : 29 } output_trace: { packet_timestamp: 2366666 @@ -4041,7 +4041,7 @@ graph_trace: { finish_time : 1944287 packet_timestamp: 2400000 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 2400000 @@ -4083,7 +4083,7 @@ graph_trace: { finish_time : 1969585 packet_timestamp: 2433333 stream_id : 2 - packet_id : 47 + event_data : 47 } output_trace: { packet_timestamp: 2433333 @@ -4125,7 +4125,7 @@ graph_trace: { finish_time : 1995205 packet_timestamp: 2466666 stream_id : 2 - packet_id : 34 + event_data : 34 } output_trace: { packet_timestamp: 2466666 @@ -4167,7 +4167,7 @@ graph_trace: { finish_time : 2020248 packet_timestamp: 2500000 stream_id : 2 - packet_id : 27 + event_data : 27 } output_trace: { packet_timestamp: 2500000 @@ -4209,7 +4209,7 @@ graph_trace: { finish_time : 2046125 packet_timestamp: 2533333 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 2533333 @@ -4238,7 +4238,7 @@ graph_trace: { finish_time : 2062786 packet_timestamp: 1533333 stream_id : 3 - packet_id : 48 + event_data : 48 } thread_id : 1 } @@ -4257,7 +4257,7 @@ graph_trace: { finish_time : 2086023 packet_timestamp: 1566666 stream_id : 3 - packet_id : 37 + event_data : 37 } thread_id : 1 } @@ -4289,7 +4289,7 @@ graph_trace: { finish_time : 2088006 packet_timestamp: 2566666 stream_id : 2 - packet_id : 29 + event_data : 29 } output_trace: { packet_timestamp: 2566666 @@ -4312,7 +4312,7 @@ graph_trace: { finish_time : 2106315 packet_timestamp: 1600000 stream_id : 3 - packet_id : 49 + event_data : 49 } thread_id : 1 } @@ -4350,7 +4350,7 @@ graph_trace: { finish_time : 2132008 packet_timestamp: 2600000 stream_id : 2 - packet_id : 37 + event_data : 37 } output_trace: { packet_timestamp: 2600000 @@ -4379,7 +4379,7 @@ graph_trace: { finish_time : 2159704 packet_timestamp: 1633333 stream_id : 3 - packet_id : 42 + event_data : 42 } thread_id : 1 } @@ -4411,7 +4411,7 @@ graph_trace: { finish_time : 2160024 packet_timestamp: 2633333 stream_id : 2 - packet_id : 27 + event_data : 27 } output_trace: { packet_timestamp: 2633333 @@ -4434,7 +4434,7 @@ graph_trace: { finish_time : 2186107 packet_timestamp: 1666666 stream_id : 3 - packet_id : 23 + event_data : 23 } thread_id : 1 } @@ -4459,7 +4459,7 @@ graph_trace: { finish_time : 2223621 packet_timestamp: 1700000 stream_id : 3 - packet_id : 41 + event_data : 41 } thread_id : 1 } @@ -4491,7 +4491,7 @@ graph_trace: { finish_time : 2226387 packet_timestamp: 2666666 stream_id : 2 - packet_id : 42 + event_data : 42 } output_trace: { packet_timestamp: 2666666 @@ -4514,7 +4514,7 @@ graph_trace: { finish_time : 2254021 packet_timestamp: 1733333 stream_id : 3 - packet_id : 50 + event_data : 50 } thread_id : 1 } @@ -4539,7 +4539,7 @@ graph_trace: { finish_time : 2285563 packet_timestamp: 1766666 stream_id : 3 - packet_id : 43 + event_data : 43 } thread_id : 1 } @@ -4571,7 +4571,7 @@ graph_trace: { finish_time : 2287921 packet_timestamp: 2700000 stream_id : 2 - packet_id : 7 + event_data : 7 } output_trace: { packet_timestamp: 2700000 @@ -4594,7 +4594,7 @@ graph_trace: { finish_time : 2312506 packet_timestamp: 1800000 stream_id : 3 - packet_id : 25 + event_data : 25 } thread_id : 1 } @@ -4619,7 +4619,7 @@ graph_trace: { finish_time : 2349846 packet_timestamp: 1833333 stream_id : 3 - packet_id : 51 + event_data : 51 } thread_id : 1 } @@ -4651,7 +4651,7 @@ graph_trace: { finish_time : 2352490 packet_timestamp: 2733333 stream_id : 2 - packet_id : 43 + event_data : 43 } output_trace: { packet_timestamp: 2733333 @@ -4685,7 +4685,7 @@ graph_trace: { finish_time : 2386192 packet_timestamp: 1866666 stream_id : 3 - packet_id : 35 + event_data : 35 } thread_id : 1 } @@ -4717,7 +4717,7 @@ graph_trace: { finish_time : 2422842 packet_timestamp: 2766666 stream_id : 2 - packet_id : 26 + event_data : 26 } output_trace: { packet_timestamp: 2766666 @@ -4740,7 +4740,7 @@ graph_trace: { finish_time : 2431293 packet_timestamp: 1900000 stream_id : 3 - packet_id : 17 + event_data : 17 } thread_id : 1 } @@ -4765,7 +4765,7 @@ graph_trace: { finish_time : 2472853 packet_timestamp: 1933333 stream_id : 3 - packet_id : 33 + event_data : 33 } thread_id : 1 } @@ -4797,7 +4797,7 @@ graph_trace: { finish_time : 2476809 packet_timestamp: 2800000 stream_id : 2 - packet_id : 35 + event_data : 35 } output_trace: { packet_timestamp: 2800000 @@ -4820,7 +4820,7 @@ graph_trace: { finish_time : 2507616 packet_timestamp: 1966666 stream_id : 3 - packet_id : 36 + event_data : 36 } thread_id : 1 } @@ -4858,7 +4858,7 @@ graph_trace: { finish_time : 2535248 packet_timestamp: 2833333 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 2833333 @@ -4900,7 +4900,7 @@ graph_trace: { finish_time : 2568350 packet_timestamp: 2866666 stream_id : 2 - packet_id : 23 + event_data : 23 } output_trace: { packet_timestamp: 2866666 @@ -4929,7 +4929,7 @@ graph_trace: { finish_time : 2605646 packet_timestamp: 2000000 stream_id : 3 - packet_id : 28 + event_data : 28 } thread_id : 1 } @@ -4961,7 +4961,7 @@ graph_trace: { finish_time : 2608741 packet_timestamp: 2900000 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 2900000 @@ -5003,7 +5003,7 @@ graph_trace: { finish_time : 2658885 packet_timestamp: 2933333 stream_id : 2 - packet_id : 25 + event_data : 25 } output_trace: { packet_timestamp: 2933333 @@ -5045,7 +5045,7 @@ graph_trace: { finish_time : 2688158 packet_timestamp: 2966666 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 2966666 @@ -5087,7 +5087,7 @@ graph_trace: { finish_time : 2721281 packet_timestamp: 3000000 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 3000000 @@ -5129,7 +5129,7 @@ graph_trace: { finish_time : 2754442 packet_timestamp: 3033333 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 3033333 @@ -5171,7 +5171,7 @@ graph_trace: { finish_time : 2787126 packet_timestamp: 3066666 stream_id : 2 - packet_id : 25 + event_data : 25 } output_trace: { packet_timestamp: 3066666 @@ -5213,7 +5213,7 @@ graph_trace: { finish_time : 2831121 packet_timestamp: 3100000 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 3100000 @@ -5254,7 +5254,7 @@ graph_trace: { finish_time : 2870165 packet_timestamp: 3133333 stream_id : 2 - packet_id : 29 + event_data : 29 } thread_id : 0 } @@ -5295,7 +5295,7 @@ graph_trace: { finish_time : 2898478 packet_timestamp: 2033333 stream_id : 3 - packet_id : 45 + event_data : 45 } thread_id : 1 } @@ -5327,7 +5327,7 @@ graph_trace: { finish_time : 2925970 packet_timestamp: 3166666 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 3166666 @@ -5369,7 +5369,7 @@ graph_trace: { finish_time : 2957751 packet_timestamp: 3200000 stream_id : 2 - packet_id : 25 + event_data : 25 } output_trace: { packet_timestamp: 3200000 @@ -5398,7 +5398,7 @@ graph_trace: { finish_time : 2984302 packet_timestamp: 2066666 stream_id : 3 - packet_id : 53 + event_data : 53 } thread_id : 1 } @@ -5430,7 +5430,7 @@ graph_trace: { finish_time : 2992719 packet_timestamp: 3233333 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 3233333 @@ -5472,7 +5472,7 @@ graph_trace: { finish_time : 3022115 packet_timestamp: 3266666 stream_id : 2 - packet_id : 54 + event_data : 54 } output_trace: { packet_timestamp: 3266666 @@ -5495,7 +5495,7 @@ graph_trace: { finish_time : 3036136 packet_timestamp: 2100000 stream_id : 3 - packet_id : 44 + event_data : 44 } thread_id : 1 } @@ -5533,7 +5533,7 @@ graph_trace: { finish_time : 3071699 packet_timestamp: 3300000 stream_id : 2 - packet_id : 55 + event_data : 55 } output_trace: { packet_timestamp: 3300000 @@ -5575,7 +5575,7 @@ graph_trace: { finish_time : 3111576 packet_timestamp: 3333333 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 3333333 @@ -5617,7 +5617,7 @@ graph_trace: { finish_time : 3155136 packet_timestamp: 3366666 stream_id : 2 - packet_id : 56 + event_data : 56 } output_trace: { packet_timestamp: 3366666 @@ -5659,7 +5659,7 @@ graph_trace: { finish_time : 3197702 packet_timestamp: 3400000 stream_id : 2 - packet_id : 54 + event_data : 54 } output_trace: { packet_timestamp: 3400000 @@ -5682,7 +5682,7 @@ graph_trace: { finish_time : 3208310 packet_timestamp: 2133333 stream_id : 3 - packet_id : 57 + event_data : 57 } thread_id : 1 } @@ -5720,7 +5720,7 @@ graph_trace: { finish_time : 3252605 packet_timestamp: 3433333 stream_id : 2 - packet_id : 44 + event_data : 44 } output_trace: { packet_timestamp: 3433333 @@ -5743,7 +5743,7 @@ graph_trace: { finish_time : 3255332 packet_timestamp: 2166666 stream_id : 3 - packet_id : 58 + event_data : 58 } thread_id : 1 } @@ -5781,7 +5781,7 @@ graph_trace: { finish_time : 3286163 packet_timestamp: 3466666 stream_id : 2 - packet_id : 57 + event_data : 57 } output_trace: { packet_timestamp: 3466666 @@ -5823,7 +5823,7 @@ graph_trace: { finish_time : 3326956 packet_timestamp: 3500000 stream_id : 2 - packet_id : 59 + event_data : 59 } output_trace: { packet_timestamp: 3500000 @@ -5852,7 +5852,7 @@ graph_trace: { finish_time : 3345688 packet_timestamp: 2200000 stream_id : 3 - packet_id : 60 + event_data : 60 } thread_id : 1 } @@ -5871,7 +5871,7 @@ graph_trace: { finish_time : 3364829 packet_timestamp: 2233333 stream_id : 3 - packet_id : 39 + event_data : 39 } thread_id : 1 } @@ -5903,7 +5903,7 @@ graph_trace: { finish_time : 3369698 packet_timestamp: 3533333 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 3533333 @@ -5950,7 +5950,7 @@ graph_trace: { finish_time : 3421713 packet_timestamp: 3566666 stream_id : 2 - packet_id : 44 + event_data : 44 } output_trace: { packet_timestamp: 3566666 @@ -5992,7 +5992,7 @@ graph_trace: { finish_time : 3458554 packet_timestamp: 3600000 stream_id : 2 - packet_id : 57 + event_data : 57 } output_trace: { packet_timestamp: 3600000 @@ -6015,7 +6015,7 @@ graph_trace: { finish_time : 3461073 packet_timestamp: 2266666 stream_id : 3 - packet_id : 61 + event_data : 61 } thread_id : 1 } @@ -6053,7 +6053,7 @@ graph_trace: { finish_time : 3494591 packet_timestamp: 3633333 stream_id : 2 - packet_id : 39 + event_data : 39 } output_trace: { packet_timestamp: 3633333 @@ -6082,7 +6082,7 @@ graph_trace: { finish_time : 3515680 packet_timestamp: 2300000 stream_id : 3 - packet_id : 62 + event_data : 62 } thread_id : 1 } @@ -6114,7 +6114,7 @@ graph_trace: { finish_time : 3541236 packet_timestamp: 3666666 stream_id : 2 - packet_id : 63 + event_data : 63 } output_trace: { packet_timestamp: 3666666 @@ -6137,7 +6137,7 @@ graph_trace: { finish_time : 3558632 packet_timestamp: 2333333 stream_id : 3 - packet_id : 10 + event_data : 10 } thread_id : 1 } @@ -6175,7 +6175,7 @@ graph_trace: { finish_time : 3578659 packet_timestamp: 3700000 stream_id : 2 - packet_id : 32 + event_data : 32 } output_trace: { packet_timestamp: 3700000 @@ -6198,7 +6198,7 @@ graph_trace: { finish_time : 3582031 packet_timestamp: 2366666 stream_id : 3 - packet_id : 46 + event_data : 46 } thread_id : 1 } @@ -6236,7 +6236,7 @@ graph_trace: { finish_time : 3613093 packet_timestamp: 3733333 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 3733333 @@ -6278,7 +6278,7 @@ graph_trace: { finish_time : 3650500 packet_timestamp: 3766666 stream_id : 2 - packet_id : 50 + event_data : 50 } output_trace: { packet_timestamp: 3766666 @@ -6307,7 +6307,7 @@ graph_trace: { finish_time : 3683049 packet_timestamp: 2400000 stream_id : 3 - packet_id : 9 + event_data : 9 } thread_id : 1 } @@ -6339,7 +6339,7 @@ graph_trace: { finish_time : 3693713 packet_timestamp: 3800000 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 3800000 @@ -6381,7 +6381,7 @@ graph_trace: { finish_time : 3731391 packet_timestamp: 3833333 stream_id : 2 - packet_id : 57 + event_data : 57 } output_trace: { packet_timestamp: 3833333 @@ -6410,7 +6410,7 @@ graph_trace: { finish_time : 3758624 packet_timestamp: 2433333 stream_id : 3 - packet_id : 14 + event_data : 14 } thread_id : 1 } @@ -6442,7 +6442,7 @@ graph_trace: { finish_time : 3763625 packet_timestamp: 3866666 stream_id : 2 - packet_id : 64 + event_data : 64 } output_trace: { packet_timestamp: 3866666 @@ -6471,7 +6471,7 @@ graph_trace: { finish_time : 3794171 packet_timestamp: 2466666 stream_id : 3 - packet_id : 40 + event_data : 40 } thread_id : 1 } @@ -6503,7 +6503,7 @@ graph_trace: { finish_time : 3812190 packet_timestamp: 3900000 stream_id : 2 - packet_id : 17 + event_data : 17 } output_trace: { packet_timestamp: 3900000 @@ -6545,7 +6545,7 @@ graph_trace: { finish_time : 3848730 packet_timestamp: 3933333 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 3933333 @@ -6586,7 +6586,7 @@ graph_trace: { finish_time : 3881340 packet_timestamp: 3966666 stream_id : 2 - packet_id : 65 + event_data : 65 } thread_id : 0 } @@ -6640,7 +6640,7 @@ graph_trace: { finish_time : 3915512 packet_timestamp: 4000000 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 4000000 @@ -6682,7 +6682,7 @@ graph_trace: { finish_time : 3949860 packet_timestamp: 4033333 stream_id : 2 - packet_id : 9 + event_data : 9 } output_trace: { packet_timestamp: 4033333 @@ -6724,7 +6724,7 @@ graph_trace: { finish_time : 4002016 packet_timestamp: 4066666 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 4066666 @@ -6766,7 +6766,7 @@ graph_trace: { finish_time : 4045868 packet_timestamp: 4100000 stream_id : 2 - packet_id : 65 + event_data : 65 } output_trace: { packet_timestamp: 4100000 @@ -6795,7 +6795,7 @@ graph_trace: { finish_time : 4069505 packet_timestamp: 2500000 stream_id : 3 - packet_id : 66 + event_data : 66 } thread_id : 1 } @@ -6827,7 +6827,7 @@ graph_trace: { finish_time : 4095171 packet_timestamp: 4133333 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 4133333 @@ -6869,7 +6869,7 @@ graph_trace: { finish_time : 4132294 packet_timestamp: 4166666 stream_id : 2 - packet_id : 9 + event_data : 9 } output_trace: { packet_timestamp: 4166666 @@ -6898,7 +6898,7 @@ graph_trace: { finish_time : 4183791 packet_timestamp: 2533333 stream_id : 3 - packet_id : 24 + event_data : 24 } thread_id : 1 } @@ -6930,7 +6930,7 @@ graph_trace: { finish_time : 4187201 packet_timestamp: 4200000 stream_id : 2 - packet_id : 67 + event_data : 67 } output_trace: { packet_timestamp: 4200000 @@ -6972,7 +6972,7 @@ graph_trace: { finish_time : 4233409 packet_timestamp: 4233333 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 4233333 @@ -7001,7 +7001,7 @@ graph_trace: { finish_time : 4256974 packet_timestamp: 2566666 stream_id : 3 - packet_id : 5 + event_data : 5 } thread_id : 1 } @@ -7033,7 +7033,7 @@ graph_trace: { finish_time : 4283358 packet_timestamp: 4266666 stream_id : 2 - packet_id : 35 + event_data : 35 } output_trace: { packet_timestamp: 4266666 @@ -7056,7 +7056,7 @@ graph_trace: { finish_time : 4283698 packet_timestamp: 2600000 stream_id : 3 - packet_id : 68 + event_data : 68 } thread_id : 1 } @@ -7094,7 +7094,7 @@ graph_trace: { finish_time : 4314457 packet_timestamp: 4300000 stream_id : 2 - packet_id : 5 + event_data : 5 } output_trace: { packet_timestamp: 4300000 @@ -7136,7 +7136,7 @@ graph_trace: { finish_time : 4347642 packet_timestamp: 4333333 stream_id : 2 - packet_id : 14 + event_data : 14 } output_trace: { packet_timestamp: 4333333 @@ -7183,7 +7183,7 @@ graph_trace: { finish_time : 4407541 packet_timestamp: 4366666 stream_id : 2 - packet_id : 60 + event_data : 60 } output_trace: { packet_timestamp: 4366666 @@ -7225,7 +7225,7 @@ graph_trace: { finish_time : 4455197 packet_timestamp: 4400000 stream_id : 2 - packet_id : 35 + event_data : 35 } output_trace: { packet_timestamp: 4400000 @@ -7248,7 +7248,7 @@ graph_trace: { finish_time : 4458305 packet_timestamp: 2633333 stream_id : 3 - packet_id : 38 + event_data : 38 } thread_id : 1 } @@ -7286,7 +7286,7 @@ graph_trace: { finish_time : 4487469 packet_timestamp: 4433333 stream_id : 2 - packet_id : 68 + event_data : 68 } output_trace: { packet_timestamp: 4433333 @@ -7328,7 +7328,7 @@ graph_trace: { finish_time : 4522877 packet_timestamp: 4466666 stream_id : 2 - packet_id : 9 + event_data : 9 } output_trace: { packet_timestamp: 4466666 @@ -7351,7 +7351,7 @@ graph_trace: { finish_time : 4526023 packet_timestamp: 2666666 stream_id : 3 - packet_id : 20 + event_data : 20 } thread_id : 1 } @@ -7389,7 +7389,7 @@ graph_trace: { finish_time : 4554578 packet_timestamp: 4500000 stream_id : 2 - packet_id : 38 + event_data : 38 } output_trace: { packet_timestamp: 4500000 @@ -7412,7 +7412,7 @@ graph_trace: { finish_time : 4562877 packet_timestamp: 2700000 stream_id : 3 - packet_id : 69 + event_data : 69 } thread_id : 1 } @@ -7450,7 +7450,7 @@ graph_trace: { finish_time : 4590596 packet_timestamp: 4533333 stream_id : 2 - packet_id : 20 + event_data : 20 } output_trace: { packet_timestamp: 4533333 @@ -7473,7 +7473,7 @@ graph_trace: { finish_time : 4596904 packet_timestamp: 2733333 stream_id : 3 - packet_id : 70 + event_data : 70 } thread_id : 1 } @@ -7511,7 +7511,7 @@ graph_trace: { finish_time : 4629015 packet_timestamp: 4566666 stream_id : 2 - packet_id : 69 + event_data : 69 } output_trace: { packet_timestamp: 4566666 @@ -7553,7 +7553,7 @@ graph_trace: { finish_time : 4670468 packet_timestamp: 4600000 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 4600000 @@ -7595,7 +7595,7 @@ graph_trace: { finish_time : 4712581 packet_timestamp: 4633333 stream_id : 2 - packet_id : 66 + event_data : 66 } output_trace: { packet_timestamp: 4633333 @@ -7637,7 +7637,7 @@ graph_trace: { finish_time : 4758617 packet_timestamp: 4666666 stream_id : 2 - packet_id : 20 + event_data : 20 } output_trace: { packet_timestamp: 4666666 @@ -7660,7 +7660,7 @@ graph_trace: { finish_time : 4770758 packet_timestamp: 2766666 stream_id : 3 - packet_id : 37 + event_data : 37 } thread_id : 1 } @@ -7698,7 +7698,7 @@ graph_trace: { finish_time : 4799727 packet_timestamp: 4700000 stream_id : 2 - packet_id : 70 + event_data : 70 } output_trace: { packet_timestamp: 4700000 @@ -7721,7 +7721,7 @@ graph_trace: { finish_time : 4810914 packet_timestamp: 2800000 stream_id : 3 - packet_id : 42 + event_data : 42 } thread_id : 1 } @@ -7759,7 +7759,7 @@ graph_trace: { finish_time : 4837280 packet_timestamp: 4733333 stream_id : 2 - packet_id : 37 + event_data : 37 } output_trace: { packet_timestamp: 4733333 @@ -7782,7 +7782,7 @@ graph_trace: { finish_time : 4852985 packet_timestamp: 2833333 stream_id : 3 - packet_id : 41 + event_data : 41 } thread_id : 1 } @@ -7819,7 +7819,7 @@ graph_trace: { finish_time : 4877749 packet_timestamp: 4766666 stream_id : 2 - packet_id : 71 + event_data : 71 } thread_id : 0 } diff --git a/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt b/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt index e174ff88d..28079ae9a 100644 --- a/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt +++ b/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt @@ -29,7 +29,7 @@ graph_trace: { finish_time : 4890249 packet_timestamp: 2866666 stream_id : 3 - packet_id : 49 + event_data : 49 } thread_id : 1 } @@ -67,7 +67,7 @@ graph_trace: { finish_time : 4912870 packet_timestamp: 4800000 stream_id : 2 - packet_id : 42 + event_data : 42 } output_trace: { packet_timestamp: 4800000 @@ -96,7 +96,7 @@ graph_trace: { finish_time : 4936076 packet_timestamp: 2900000 stream_id : 3 - packet_id : 27 + event_data : 27 } thread_id : 1 } @@ -128,7 +128,7 @@ graph_trace: { finish_time : 4963567 packet_timestamp: 4833333 stream_id : 2 - packet_id : 37 + event_data : 37 } output_trace: { packet_timestamp: 4833333 @@ -151,7 +151,7 @@ graph_trace: { finish_time : 4978624 packet_timestamp: 2933333 stream_id : 3 - packet_id : 51 + event_data : 51 } thread_id : 1 } @@ -189,7 +189,7 @@ graph_trace: { finish_time : 5006617 packet_timestamp: 4866666 stream_id : 2 - packet_id : 72 + event_data : 72 } output_trace: { packet_timestamp: 4866666 @@ -231,7 +231,7 @@ graph_trace: { finish_time : 5053000 packet_timestamp: 4900000 stream_id : 2 - packet_id : 73 + event_data : 73 } output_trace: { packet_timestamp: 4900000 @@ -260,7 +260,7 @@ graph_trace: { finish_time : 5074302 packet_timestamp: 2966666 stream_id : 3 - packet_id : 7 + event_data : 7 } thread_id : 1 } @@ -292,7 +292,7 @@ graph_trace: { finish_time : 5099137 packet_timestamp: 4933333 stream_id : 2 - packet_id : 71 + event_data : 71 } output_trace: { packet_timestamp: 4933333 @@ -315,7 +315,7 @@ graph_trace: { finish_time : 5108979 packet_timestamp: 3000000 stream_id : 3 - packet_id : 31 + event_data : 31 } thread_id : 1 } @@ -353,7 +353,7 @@ graph_trace: { finish_time : 5133956 packet_timestamp: 4966666 stream_id : 2 - packet_id : 7 + event_data : 7 } output_trace: { packet_timestamp: 4966666 @@ -395,7 +395,7 @@ graph_trace: { finish_time : 5166537 packet_timestamp: 5000000 stream_id : 2 - packet_id : 72 + event_data : 72 } output_trace: { packet_timestamp: 5000000 @@ -424,7 +424,7 @@ graph_trace: { finish_time : 5212613 packet_timestamp: 3033333 stream_id : 3 - packet_id : 74 + event_data : 74 } thread_id : 1 } @@ -456,7 +456,7 @@ graph_trace: { finish_time : 5214212 packet_timestamp: 5033333 stream_id : 2 - packet_id : 75 + event_data : 75 } output_trace: { packet_timestamp: 5033333 @@ -485,7 +485,7 @@ graph_trace: { finish_time : 5248772 packet_timestamp: 3066666 stream_id : 3 - packet_id : 16 + event_data : 16 } thread_id : 1 } @@ -517,7 +517,7 @@ graph_trace: { finish_time : 5272571 packet_timestamp: 5066666 stream_id : 2 - packet_id : 31 + event_data : 31 } output_trace: { packet_timestamp: 5066666 @@ -546,7 +546,7 @@ graph_trace: { finish_time : 5307543 packet_timestamp: 3100000 stream_id : 3 - packet_id : 76 + event_data : 76 } thread_id : 1 } @@ -578,7 +578,7 @@ graph_trace: { finish_time : 5336031 packet_timestamp: 5100000 stream_id : 2 - packet_id : 77 + event_data : 77 } output_trace: { packet_timestamp: 5100000 @@ -620,7 +620,7 @@ graph_trace: { finish_time : 5363534 packet_timestamp: 5133333 stream_id : 2 - packet_id : 71 + event_data : 71 } output_trace: { packet_timestamp: 5133333 @@ -667,7 +667,7 @@ graph_trace: { finish_time : 5394729 packet_timestamp: 5166666 stream_id : 2 - packet_id : 74 + event_data : 74 } output_trace: { packet_timestamp: 5166666 @@ -709,7 +709,7 @@ graph_trace: { finish_time : 5427127 packet_timestamp: 5200000 stream_id : 2 - packet_id : 73 + event_data : 73 } output_trace: { packet_timestamp: 5200000 @@ -751,7 +751,7 @@ graph_trace: { finish_time : 5486066 packet_timestamp: 5233333 stream_id : 2 - packet_id : 77 + event_data : 77 } output_trace: { packet_timestamp: 5233333 @@ -793,7 +793,7 @@ graph_trace: { finish_time : 5518571 packet_timestamp: 5266666 stream_id : 2 - packet_id : 78 + event_data : 78 } output_trace: { packet_timestamp: 5266666 @@ -835,7 +835,7 @@ graph_trace: { finish_time : 5554239 packet_timestamp: 5300000 stream_id : 2 - packet_id : 79 + event_data : 79 } output_trace: { packet_timestamp: 5300000 @@ -858,7 +858,7 @@ graph_trace: { finish_time : 5555750 packet_timestamp: 3133333 stream_id : 3 - packet_id : 80 + event_data : 80 } thread_id : 1 } @@ -896,7 +896,7 @@ graph_trace: { finish_time : 5582215 packet_timestamp: 5333333 stream_id : 2 - packet_id : 76 + event_data : 76 } output_trace: { packet_timestamp: 5333333 @@ -919,7 +919,7 @@ graph_trace: { finish_time : 5602676 packet_timestamp: 3166666 stream_id : 3 - packet_id : 3 + event_data : 3 } thread_id : 1 } @@ -944,7 +944,7 @@ graph_trace: { finish_time : 5636894 packet_timestamp: 3200000 stream_id : 3 - packet_id : 81 + event_data : 81 } thread_id : 1 } @@ -976,7 +976,7 @@ graph_trace: { finish_time : 5665051 packet_timestamp: 5366666 stream_id : 2 - packet_id : 80 + event_data : 80 } output_trace: { packet_timestamp: 5366666 @@ -999,7 +999,7 @@ graph_trace: { finish_time : 5682062 packet_timestamp: 3233333 stream_id : 3 - packet_id : 82 + event_data : 82 } thread_id : 1 } @@ -1037,7 +1037,7 @@ graph_trace: { finish_time : 5726399 packet_timestamp: 5400000 stream_id : 2 - packet_id : 83 + event_data : 83 } output_trace: { packet_timestamp: 5400000 @@ -1066,7 +1066,7 @@ graph_trace: { finish_time : 5768290 packet_timestamp: 3266666 stream_id : 3 - packet_id : 23 + event_data : 23 } thread_id : 1 } @@ -1098,7 +1098,7 @@ graph_trace: { finish_time : 5774718 packet_timestamp: 5433333 stream_id : 2 - packet_id : 11 + event_data : 11 } output_trace: { packet_timestamp: 5433333 @@ -1140,7 +1140,7 @@ graph_trace: { finish_time : 5820969 packet_timestamp: 5466666 stream_id : 2 - packet_id : 73 + event_data : 73 } output_trace: { packet_timestamp: 5466666 @@ -1163,7 +1163,7 @@ graph_trace: { finish_time : 5823620 packet_timestamp: 3300000 stream_id : 3 - packet_id : 84 + event_data : 84 } thread_id : 1 } @@ -1201,7 +1201,7 @@ graph_trace: { finish_time : 5856982 packet_timestamp: 5500000 stream_id : 2 - packet_id : 23 + event_data : 23 } output_trace: { packet_timestamp: 5500000 @@ -1235,7 +1235,7 @@ graph_trace: { finish_time : 5887619 packet_timestamp: 3333333 stream_id : 3 - packet_id : 52 + event_data : 52 } thread_id : 1 } @@ -1267,7 +1267,7 @@ graph_trace: { finish_time : 5893613 packet_timestamp: 5533333 stream_id : 2 - packet_id : 80 + event_data : 80 } output_trace: { packet_timestamp: 5533333 @@ -1313,7 +1313,7 @@ graph_trace: { finish_time : 5930578 packet_timestamp: 3366666 stream_id : 3 - packet_id : 53 + event_data : 53 } thread_id : 1 } @@ -1328,7 +1328,7 @@ graph_trace: { finish_time : 5930579 packet_timestamp: 5566666 stream_id : 2 - packet_id : 82 + event_data : 82 } output_trace: { packet_timestamp: 5566666 @@ -1370,7 +1370,7 @@ graph_trace: { finish_time : 5990784 packet_timestamp: 5600000 stream_id : 2 - packet_id : 73 + event_data : 73 } output_trace: { packet_timestamp: 5600000 @@ -1399,7 +1399,7 @@ graph_trace: { finish_time : 6015925 packet_timestamp: 3400000 stream_id : 3 - packet_id : 45 + event_data : 45 } thread_id : 1 } @@ -1431,7 +1431,7 @@ graph_trace: { finish_time : 6034123 packet_timestamp: 5633333 stream_id : 2 - packet_id : 82 + event_data : 82 } output_trace: { packet_timestamp: 5633333 @@ -1454,7 +1454,7 @@ graph_trace: { finish_time : 6041106 packet_timestamp: 3433333 stream_id : 3 - packet_id : 85 + event_data : 85 } thread_id : 1 } @@ -1492,7 +1492,7 @@ graph_trace: { finish_time : 6065336 packet_timestamp: 5666666 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 5666666 @@ -1534,7 +1534,7 @@ graph_trace: { finish_time : 6102574 packet_timestamp: 5700000 stream_id : 2 - packet_id : 86 + event_data : 86 } output_trace: { packet_timestamp: 5700000 @@ -1563,7 +1563,7 @@ graph_trace: { finish_time : 6128837 packet_timestamp: 3466666 stream_id : 3 - packet_id : 33 + event_data : 33 } thread_id : 1 } @@ -1595,7 +1595,7 @@ graph_trace: { finish_time : 6164292 packet_timestamp: 5733333 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 5733333 @@ -1618,7 +1618,7 @@ graph_trace: { finish_time : 6171826 packet_timestamp: 3500000 stream_id : 3 - packet_id : 87 + event_data : 87 } thread_id : 1 } @@ -1656,7 +1656,7 @@ graph_trace: { finish_time : 6213894 packet_timestamp: 5766666 stream_id : 2 - packet_id : 33 + event_data : 33 } output_trace: { packet_timestamp: 5766666 @@ -1698,7 +1698,7 @@ graph_trace: { finish_time : 6258767 packet_timestamp: 5800000 stream_id : 2 - packet_id : 88 + event_data : 88 } output_trace: { packet_timestamp: 5800000 @@ -1727,7 +1727,7 @@ graph_trace: { finish_time : 6298470 packet_timestamp: 3533333 stream_id : 3 - packet_id : 89 + event_data : 89 } thread_id : 1 } @@ -1759,7 +1759,7 @@ graph_trace: { finish_time : 6302123 packet_timestamp: 5833333 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 5833333 @@ -1801,7 +1801,7 @@ graph_trace: { finish_time : 6336724 packet_timestamp: 5866666 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 5866666 @@ -1842,7 +1842,7 @@ graph_trace: { finish_time : 6378613 packet_timestamp: 5900000 stream_id : 2 - packet_id : 42 + event_data : 42 } thread_id : 0 } @@ -1883,7 +1883,7 @@ graph_trace: { finish_time : 6407444 packet_timestamp: 3566666 stream_id : 3 - packet_id : 43 + event_data : 43 } thread_id : 1 } @@ -1915,7 +1915,7 @@ graph_trace: { finish_time : 6418588 packet_timestamp: 5933333 stream_id : 2 - packet_id : 87 + event_data : 87 } output_trace: { packet_timestamp: 5933333 @@ -1957,7 +1957,7 @@ graph_trace: { finish_time : 6447815 packet_timestamp: 5966666 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 5966666 @@ -1999,7 +1999,7 @@ graph_trace: { finish_time : 6482548 packet_timestamp: 6000000 stream_id : 2 - packet_id : 45 + event_data : 45 } output_trace: { packet_timestamp: 6000000 @@ -2028,7 +2028,7 @@ graph_trace: { finish_time : 6502362 packet_timestamp: 3600000 stream_id : 3 - packet_id : 8 + event_data : 8 } thread_id : 1 } @@ -2060,7 +2060,7 @@ graph_trace: { finish_time : 6519040 packet_timestamp: 6033333 stream_id : 2 - packet_id : 52 + event_data : 52 } output_trace: { packet_timestamp: 6033333 @@ -2089,7 +2089,7 @@ graph_trace: { finish_time : 6537106 packet_timestamp: 3633333 stream_id : 3 - packet_id : 47 + event_data : 47 } thread_id : 1 } @@ -2121,7 +2121,7 @@ graph_trace: { finish_time : 6577536 packet_timestamp: 6066666 stream_id : 2 - packet_id : 90 + event_data : 90 } output_trace: { packet_timestamp: 6066666 @@ -2163,7 +2163,7 @@ graph_trace: { finish_time : 6612170 packet_timestamp: 6100000 stream_id : 2 - packet_id : 91 + event_data : 91 } output_trace: { packet_timestamp: 6100000 @@ -2205,7 +2205,7 @@ graph_trace: { finish_time : 6659509 packet_timestamp: 6133333 stream_id : 2 - packet_id : 36 + event_data : 36 } output_trace: { packet_timestamp: 6133333 @@ -2247,7 +2247,7 @@ graph_trace: { finish_time : 6696591 packet_timestamp: 6166666 stream_id : 2 - packet_id : 92 + event_data : 92 } output_trace: { packet_timestamp: 6166666 @@ -2289,7 +2289,7 @@ graph_trace: { finish_time : 6733330 packet_timestamp: 6200000 stream_id : 2 - packet_id : 90 + event_data : 90 } output_trace: { packet_timestamp: 6200000 @@ -2331,7 +2331,7 @@ graph_trace: { finish_time : 6769493 packet_timestamp: 6233333 stream_id : 2 - packet_id : 91 + event_data : 91 } output_trace: { packet_timestamp: 6233333 @@ -2373,7 +2373,7 @@ graph_trace: { finish_time : 6808565 packet_timestamp: 6266666 stream_id : 2 - packet_id : 83 + event_data : 83 } output_trace: { packet_timestamp: 6266666 @@ -2415,7 +2415,7 @@ graph_trace: { finish_time : 6845130 packet_timestamp: 6300000 stream_id : 2 - packet_id : 92 + event_data : 92 } output_trace: { packet_timestamp: 6300000 @@ -2456,7 +2456,7 @@ graph_trace: { finish_time : 6886034 packet_timestamp: 6333333 stream_id : 2 - packet_id : 74 + event_data : 74 } thread_id : 0 } @@ -2510,7 +2510,7 @@ graph_trace: { finish_time : 6929273 packet_timestamp: 6366666 stream_id : 2 - packet_id : 91 + event_data : 91 } output_trace: { packet_timestamp: 6366666 @@ -2539,7 +2539,7 @@ graph_trace: { finish_time : 6952248 packet_timestamp: 3666666 stream_id : 3 - packet_id : 93 + event_data : 93 } thread_id : 1 } @@ -2571,7 +2571,7 @@ graph_trace: { finish_time : 6977536 packet_timestamp: 6400000 stream_id : 2 - packet_id : 83 + event_data : 83 } output_trace: { packet_timestamp: 6400000 @@ -2613,7 +2613,7 @@ graph_trace: { finish_time : 7008835 packet_timestamp: 6433333 stream_id : 2 - packet_id : 94 + event_data : 94 } output_trace: { packet_timestamp: 6433333 @@ -2642,7 +2642,7 @@ graph_trace: { finish_time : 7039702 packet_timestamp: 3700000 stream_id : 3 - packet_id : 62 + event_data : 62 } thread_id : 1 } @@ -2674,7 +2674,7 @@ graph_trace: { finish_time : 7041966 packet_timestamp: 6466666 stream_id : 2 - packet_id : 74 + event_data : 74 } output_trace: { packet_timestamp: 6466666 @@ -2697,7 +2697,7 @@ graph_trace: { finish_time : 7060314 packet_timestamp: 3733333 stream_id : 3 - packet_id : 25 + event_data : 25 } thread_id : 1 } @@ -2722,7 +2722,7 @@ graph_trace: { finish_time : 7085011 packet_timestamp: 3766666 stream_id : 3 - packet_id : 32 + event_data : 32 } thread_id : 1 } @@ -2754,7 +2754,7 @@ graph_trace: { finish_time : 7091780 packet_timestamp: 6500000 stream_id : 2 - packet_id : 62 + event_data : 62 } output_trace: { packet_timestamp: 6500000 @@ -2783,7 +2783,7 @@ graph_trace: { finish_time : 7127711 packet_timestamp: 3800000 stream_id : 3 - packet_id : 95 + event_data : 95 } thread_id : 1 } @@ -2815,7 +2815,7 @@ graph_trace: { finish_time : 7128033 packet_timestamp: 6533333 stream_id : 2 - packet_id : 83 + event_data : 83 } output_trace: { packet_timestamp: 6533333 @@ -2838,7 +2838,7 @@ graph_trace: { finish_time : 7149694 packet_timestamp: 3833333 stream_id : 3 - packet_id : 46 + event_data : 46 } thread_id : 1 } @@ -2876,7 +2876,7 @@ graph_trace: { finish_time : 7168073 packet_timestamp: 6566666 stream_id : 2 - packet_id : 96 + event_data : 96 } output_trace: { packet_timestamp: 6566666 @@ -2899,7 +2899,7 @@ graph_trace: { finish_time : 7179915 packet_timestamp: 3866666 stream_id : 3 - packet_id : 97 + event_data : 97 } thread_id : 1 } @@ -2924,7 +2924,7 @@ graph_trace: { finish_time : 7216907 packet_timestamp: 3900000 stream_id : 3 - packet_id : 64 + event_data : 64 } thread_id : 1 } @@ -2956,7 +2956,7 @@ graph_trace: { finish_time : 7217293 packet_timestamp: 6600000 stream_id : 2 - packet_id : 98 + event_data : 98 } output_trace: { packet_timestamp: 6600000 @@ -2985,7 +2985,7 @@ graph_trace: { finish_time : 7259223 packet_timestamp: 3933333 stream_id : 3 - packet_id : 57 + event_data : 57 } thread_id : 1 } @@ -3017,7 +3017,7 @@ graph_trace: { finish_time : 7266595 packet_timestamp: 6633333 stream_id : 2 - packet_id : 99 + event_data : 99 } output_trace: { packet_timestamp: 6633333 @@ -3059,7 +3059,7 @@ graph_trace: { finish_time : 7298513 packet_timestamp: 6666666 stream_id : 2 - packet_id : 100 + event_data : 100 } output_trace: { packet_timestamp: 6666666 @@ -3088,7 +3088,7 @@ graph_trace: { finish_time : 7318592 packet_timestamp: 3966666 stream_id : 3 - packet_id : 101 + event_data : 101 } thread_id : 1 } @@ -3120,7 +3120,7 @@ graph_trace: { finish_time : 7328296 packet_timestamp: 6700000 stream_id : 2 - packet_id : 97 + event_data : 97 } output_trace: { packet_timestamp: 6700000 @@ -3162,7 +3162,7 @@ graph_trace: { finish_time : 7361385 packet_timestamp: 6733333 stream_id : 2 - packet_id : 102 + event_data : 102 } output_trace: { packet_timestamp: 6733333 @@ -3185,7 +3185,7 @@ graph_trace: { finish_time : 7367687 packet_timestamp: 4000000 stream_id : 3 - packet_id : 10 + event_data : 10 } thread_id : 1 } @@ -3228,7 +3228,7 @@ graph_trace: { finish_time : 7409273 packet_timestamp: 6766666 stream_id : 2 - packet_id : 101 + event_data : 101 } output_trace: { packet_timestamp: 6766666 @@ -3251,7 +3251,7 @@ graph_trace: { finish_time : 7414367 packet_timestamp: 4033333 stream_id : 3 - packet_id : 29 + event_data : 29 } thread_id : 1 } @@ -3276,7 +3276,7 @@ graph_trace: { finish_time : 7449901 packet_timestamp: 4066666 stream_id : 3 - packet_id : 30 + event_data : 30 } thread_id : 1 } @@ -3295,7 +3295,7 @@ graph_trace: { finish_time : 7483769 packet_timestamp: 4100000 stream_id : 3 - packet_id : 1 + event_data : 1 } thread_id : 1 } @@ -3327,7 +3327,7 @@ graph_trace: { finish_time : 7491738 packet_timestamp: 6800000 stream_id : 2 - packet_id : 10 + event_data : 10 } output_trace: { packet_timestamp: 6800000 @@ -3362,7 +3362,7 @@ graph_trace: { finish_time : 7713174 packet_timestamp: 4133333 stream_id : 3 - packet_id : 22 + event_data : 22 } thread_id : 0 } @@ -3381,7 +3381,7 @@ graph_trace: { finish_time : 7810691 packet_timestamp: 4166666 stream_id : 3 - packet_id : 103 + event_data : 103 } thread_id : 1 } @@ -3400,7 +3400,7 @@ graph_trace: { finish_time : 7831880 packet_timestamp: 4200000 stream_id : 3 - packet_id : 34 + event_data : 34 } thread_id : 0 } @@ -3419,7 +3419,7 @@ graph_trace: { finish_time : 7857271 packet_timestamp: 4233333 stream_id : 3 - packet_id : 104 + event_data : 104 } thread_id : 1 } @@ -3438,7 +3438,7 @@ graph_trace: { finish_time : 7878601 packet_timestamp: 4266666 stream_id : 3 - packet_id : 24 + event_data : 24 } thread_id : 0 } @@ -3462,7 +3462,7 @@ graph_trace: { finish_time : 7901813 packet_timestamp: 4300000 stream_id : 3 - packet_id : 67 + event_data : 67 } thread_id : 1 } @@ -3481,7 +3481,7 @@ graph_trace: { finish_time : 7967897 packet_timestamp: 4333333 stream_id : 3 - packet_id : 105 + event_data : 105 } thread_id : 0 } @@ -3500,7 +3500,7 @@ graph_trace: { finish_time : 8014466 packet_timestamp: 4366666 stream_id : 3 - packet_id : 106 + event_data : 106 } thread_id : 0 } @@ -3519,7 +3519,7 @@ graph_trace: { finish_time : 8171718 packet_timestamp: 4400000 stream_id : 3 - packet_id : 60 + event_data : 60 } thread_id : 1 } @@ -3538,7 +3538,7 @@ graph_trace: { finish_time : 8209047 packet_timestamp: 4433333 stream_id : 3 - packet_id : 40 + event_data : 40 } thread_id : 0 } @@ -3557,7 +3557,7 @@ graph_trace: { finish_time : 8252072 packet_timestamp: 4466666 stream_id : 3 - packet_id : 48 + event_data : 48 } thread_id : 0 } @@ -3576,7 +3576,7 @@ graph_trace: { finish_time : 8317029 packet_timestamp: 4500000 stream_id : 3 - packet_id : 19 + event_data : 19 } thread_id : 1 } @@ -3595,7 +3595,7 @@ graph_trace: { finish_time : 8355408 packet_timestamp: 4533333 stream_id : 3 - packet_id : 38 + event_data : 38 } thread_id : 0 } @@ -3619,7 +3619,7 @@ graph_trace: { finish_time : 8396832 packet_timestamp: 4566666 stream_id : 3 - packet_id : 17 + event_data : 17 } thread_id : 0 } @@ -3638,7 +3638,7 @@ graph_trace: { finish_time : 8432860 packet_timestamp: 4600000 stream_id : 3 - packet_id : 50 + event_data : 50 } thread_id : 1 } @@ -3657,7 +3657,7 @@ graph_trace: { finish_time : 8743853 packet_timestamp: 4633333 stream_id : 3 - packet_id : 63 + event_data : 63 } thread_id : 0 } @@ -3670,7 +3670,7 @@ graph_trace: { finish_time : 8787092 packet_timestamp: 4666666 stream_id : 3 - packet_id : 107 + event_data : 107 } thread_id : 1 } @@ -3689,7 +3689,7 @@ graph_trace: { finish_time : 8868426 packet_timestamp: 4700000 stream_id : 3 - packet_id : 108 + event_data : 108 } thread_id : 0 } @@ -3708,7 +3708,7 @@ graph_trace: { finish_time : 8890592 packet_timestamp: 4733333 stream_id : 3 - packet_id : 109 + event_data : 109 } thread_id : 1 } @@ -3732,7 +3732,7 @@ graph_trace: { finish_time : 8926773 packet_timestamp: 4766666 stream_id : 3 - packet_id : 110 + event_data : 110 } thread_id : 0 } @@ -3757,7 +3757,7 @@ graph_trace: { finish_time : 8948914 packet_timestamp: 4800000 stream_id : 3 - packet_id : 41 + event_data : 41 } thread_id : 1 } @@ -3776,7 +3776,7 @@ graph_trace: { finish_time : 8968938 packet_timestamp: 4833333 stream_id : 3 - packet_id : 111 + event_data : 111 } thread_id : 0 } @@ -3795,7 +3795,7 @@ graph_trace: { finish_time : 9005013 packet_timestamp: 4866666 stream_id : 3 - packet_id : 68 + event_data : 68 } thread_id : 1 } @@ -3814,7 +3814,7 @@ graph_trace: { finish_time : 9048751 packet_timestamp: 4900000 stream_id : 3 - packet_id : 112 + event_data : 112 } thread_id : 0 } @@ -3827,7 +3827,7 @@ graph_trace: { finish_time : 9081720 packet_timestamp: 4933333 stream_id : 3 - packet_id : 113 + event_data : 113 } thread_id : 1 } @@ -3852,7 +3852,7 @@ graph_trace: { finish_time : 9183973 packet_timestamp: 4966666 stream_id : 3 - packet_id : 114 + event_data : 114 } thread_id : 0 } @@ -3871,7 +3871,7 @@ graph_trace: { finish_time : 9222274 packet_timestamp: 5000000 stream_id : 3 - packet_id : 65 + event_data : 65 } thread_id : 1 } @@ -3895,7 +3895,7 @@ graph_trace: { finish_time : 9556273 packet_timestamp: 5033333 stream_id : 3 - packet_id : 115 + event_data : 115 } thread_id : 1 } @@ -3914,7 +3914,7 @@ graph_trace: { finish_time : 9650983 packet_timestamp: 5066666 stream_id : 3 - packet_id : 75 + event_data : 75 } thread_id : 0 } @@ -3933,7 +3933,7 @@ graph_trace: { finish_time : 9721061 packet_timestamp: 5100000 stream_id : 3 - packet_id : 116 + event_data : 116 } thread_id : 0 } @@ -3946,7 +3946,7 @@ graph_trace: { finish_time : 9762838 packet_timestamp: 5133333 stream_id : 3 - packet_id : 117 + event_data : 117 } thread_id : 1 } diff --git a/mediapipe/framework/profiler/trace_buffer.h b/mediapipe/framework/profiler/trace_buffer.h index c435d0d52..069f09610 100644 --- a/mediapipe/framework/profiler/trace_buffer.h +++ b/mediapipe/framework/profiler/trace_buffer.h @@ -23,45 +23,29 @@ namespace mediapipe { -// Packet content identifier. -using PacketDataId = const void*; - namespace packet_internal { -// Returns the packet data address for a packet data holder. -inline const void* GetPacketDataId(const HolderBase* holder) { - return (holder == nullptr) - ? nullptr - : &(static_cast*>(holder)->data()); +// Returns a hash of the packet data address from a packet data holder. +inline const int64 GetPacketDataId(const HolderBase* holder) { + if (holder == nullptr) { + return 0; + } + const void* address = &(static_cast*>(holder)->data()); + return reinterpret_cast(address); } } // namespace packet_internal // Packet trace log event. struct TraceEvent { using EventType = GraphTrace::EventType; - // GraphTrace::EventType constants, repeated here to match GraphProfilerStub. - static constexpr EventType UNKNOWN = GraphTrace::UNKNOWN; - static constexpr EventType OPEN = GraphTrace::OPEN; - static constexpr EventType PROCESS = GraphTrace::PROCESS; - static constexpr EventType CLOSE = GraphTrace::CLOSE; - static constexpr EventType NOT_READY = GraphTrace::NOT_READY; - static constexpr EventType READY_FOR_PROCESS = GraphTrace::READY_FOR_PROCESS; - static constexpr EventType READY_FOR_CLOSE = GraphTrace::READY_FOR_CLOSE; - static constexpr EventType THROTTLED = GraphTrace::THROTTLED; - static constexpr EventType UNTHROTTLED = GraphTrace::UNTHROTTLED; - static constexpr EventType CPU_TASK_USER = GraphTrace::CPU_TASK_USER; - static constexpr EventType CPU_TASK_SYSTEM = GraphTrace::CPU_TASK_SYSTEM; - static constexpr EventType GPU_TASK = GraphTrace::GPU_TASK; - static constexpr EventType DSP_TASK = GraphTrace::DSP_TASK; - static constexpr EventType TPU_TASK = GraphTrace::TPU_TASK; absl::Time event_time; EventType event_type = UNKNOWN; bool is_finish = false; Timestamp input_ts = Timestamp::Unset(); Timestamp packet_ts = Timestamp::Unset(); - int node_id = -1; + int32 node_id = -1; const std::string* stream_id = nullptr; - PacketDataId packet_data_id = 0; - int thread_id = 0; + int32 thread_id = 0; + int64 event_data = 0; TraceEvent(const EventType& event_type) : event_type(event_type) {} TraceEvent() {} @@ -91,7 +75,7 @@ struct TraceEvent { return *this; } inline TraceEvent& set_packet_data_id(const Packet* packet) { - this->packet_data_id = + this->event_data = packet_internal::GetPacketDataId(packet_internal::GetHolder(*packet)); return *this; } @@ -103,11 +87,84 @@ struct TraceEvent { this->is_finish = is_finish; return *this; } + inline TraceEvent& set_event_data(int data) { + this->event_data = data; + return *this; + } + + // GraphTrace::EventType constants, repeated here to match GraphProfilerStub. + static constexpr EventType UNKNOWN = GraphTrace::UNKNOWN; + static constexpr EventType OPEN = GraphTrace::OPEN; + static constexpr EventType PROCESS = GraphTrace::PROCESS; + static constexpr EventType CLOSE = GraphTrace::CLOSE; + static constexpr EventType NOT_READY = GraphTrace::NOT_READY; + static constexpr EventType READY_FOR_PROCESS = GraphTrace::READY_FOR_PROCESS; + static constexpr EventType READY_FOR_CLOSE = GraphTrace::READY_FOR_CLOSE; + static constexpr EventType THROTTLED = GraphTrace::THROTTLED; + static constexpr EventType UNTHROTTLED = GraphTrace::UNTHROTTLED; + static constexpr EventType CPU_TASK_USER = GraphTrace::CPU_TASK_USER; + static constexpr EventType CPU_TASK_SYSTEM = GraphTrace::CPU_TASK_SYSTEM; + static constexpr EventType GPU_TASK = GraphTrace::GPU_TASK; + static constexpr EventType DSP_TASK = GraphTrace::DSP_TASK; + static constexpr EventType TPU_TASK = GraphTrace::TPU_TASK; + static constexpr EventType GPU_CALIBRATION = GraphTrace::GPU_CALIBRATION; + static constexpr EventType PACKET_QUEUED = GraphTrace::PACKET_QUEUED; }; // Packet trace log buffer. using TraceBuffer = CircularBuffer; +// TraceEvent type traits. +class TraceEventType { + using EventType = TraceEvent::EventType; + + public: + TraceEventType() {} + TraceEventType(EventType event_type, std::string description, + bool is_packet_event = false, bool is_stream_event = false, + bool id_event_data = true) + : event_type_(event_type), + description_(description), + is_packet_event_(is_packet_event), + is_stream_event_(is_stream_event), + id_event_data_(id_event_data) {} + + // The type of event to log. + inline EventType event_type() const { return event_type_; } + + // True if this type of event is logged. + inline bool enabled() const { return event_type_; } + inline void set_enabled(bool enabled) { enabled_ = enabled; } + + // True if packet details are logged with this type of event. + inline bool is_packet_event() const { return is_packet_event_; } + + // True if stream details are logged with this type of event. + inline bool is_stream_event() const { return is_stream_event_; } + + // True if event_data values are assigned compact id's. + inline bool id_event_data() const { return id_event_data_; } + + private: + EventType event_type_ = TraceEvent::UNKNOWN; + std::string description_ = ""; + bool enabled_ = true; + bool is_packet_event_ = false; + bool is_stream_event_ = false; + bool id_event_data_ = true; +}; + +// A hash function for TraceEvent::EventType. +struct EventTypeHash { + size_t operator()(const TraceEvent::EventType e) const { + return static_cast(e); + } +}; + +// The registry of trace event types. +using TraceEventRegistry = + std::unordered_map; + } // namespace mediapipe #endif // MEDIAPIPE_FRAMEWORK_PROFILER_TRACE_BUFFER_H_ diff --git a/mediapipe/framework/profiler/trace_builder.cc b/mediapipe/framework/profiler/trace_builder.cc index 197472b32..c33e14b8f 100644 --- a/mediapipe/framework/profiler/trace_builder.cc +++ b/mediapipe/framework/profiler/trace_builder.cc @@ -57,21 +57,37 @@ namespace mediapipe { namespace { -// For each event-type, whether packet details are logged. -// The event-types are: -// UNKNOWN, OPEN, PROCESS, CLOSE, -// NOT_READY, READY_FOR_PROCESS, READY_FOR_CLOSE, THROTTLED, UNTHROTTLED -// CPU_TASK_USER, CPU_TASK_SYSTEM, GPU_TASK, DSP_TASK, TPU_TASK -constexpr bool kProfilerPacketEvents[] = { // - false, true, true, true, // - false, false, false, false, false, // - true, true, true, true, true}; +void BasicTraceEventTypes(TraceEventRegistry* result) { + // The initializer arguments below are: event_type, description, + // is_packet_event, is_stream_event, id_event_data. + std::vector basic_types = { + {TraceEvent::UNKNOWN, "An uninitialized trace-event."}, + {TraceEvent::OPEN, "A call to Calculator::Open.", true, true}, + {TraceEvent::PROCESS, "A call to Calculator::Open.", true, true}, + {TraceEvent::CLOSE, "A call to Calculator::Close.", true, true}, -// For each calculator method, whether StreamTraces are desired. -constexpr bool kProfilerStreamEvents[] = { // - false, true, true, true, // - false, false, false, false, false, // - true, true, false, false, false}; + {TraceEvent::NOT_READY, "A calculator cannot process packets yet."}, + {TraceEvent::READY_FOR_PROCESS, "A calculator can process packets."}, + {TraceEvent::READY_FOR_CLOSE, "A calculator is done processing packets."}, + {TraceEvent::THROTTLED, "Input is disabled due to max_queue_size."}, + {TraceEvent::UNTHROTTLED, "Input is enabled up to max_queue_size."}, + + {TraceEvent::CPU_TASK_USER, "User-time processing packets.", true, true}, + {TraceEvent::CPU_TASK_SYSTEM, "System-time processing packets.", true, + true}, + {TraceEvent::GPU_TASK, "GPU-time processing packets.", true, false}, + {TraceEvent::DSP_TASK, "DSP-time processing packets.", true, false}, + {TraceEvent::TPU_TASK, "TPU-time processing packets.", true, false}, + + {TraceEvent::GPU_CALIBRATION, + "A time measured by GPU clock and by CPU clock.", true, false}, + {TraceEvent::PACKET_QUEUED, "An input queue size when a packet arrives.", + true, true, false}, + }; + for (TraceEventType t : basic_types) { + (*result)[t.event_type()] = t; + } +} // A map defining int32 identifiers for std::string object pointers. // Lookup is fast when the same std::string object is used frequently. @@ -106,7 +122,7 @@ class StringIdMap { // A map defining int32 identifiers for object pointers. class AddressIdMap { public: - int32 operator[](const void* id) { + int32 operator[](int64 id) { auto pointer_id = pointer_id_map_.find(id); if (pointer_id != pointer_id_map_.end()) { return pointer_id->second; @@ -114,12 +130,10 @@ class AddressIdMap { return pointer_id_map_[id] = next_id++; } void clear() { pointer_id_map_.clear(); } - const std::unordered_map& map() { - return pointer_id_map_; - } + const std::unordered_map& map() { return pointer_id_map_; } private: - std::unordered_map pointer_id_map_; + std::unordered_map pointer_id_map_; int32 next_id = 0; }; @@ -147,8 +161,12 @@ class TraceBuilder::Impl { static std::string* empty_string = new std::string(""); stream_id_map_[empty_string]; packet_data_id_map_[0]; + BasicTraceEventTypes(&trace_event_registry_); } + // Returns the registry of trace event types. + TraceEventRegistry* trace_event_registry() { return &trace_event_registry_; } + static Timestamp TimestampAfter(const TraceBuffer& buffer, absl::Time begin_time) { Timestamp max_ts = Timestamp::Min(); @@ -176,7 +194,7 @@ class TraceBuilder::Impl { // Index TraceEvents by task-id and stream-hop-id. for (const TraceEvent& event : snapshot) { - if (!kProfilerPacketEvents[static_cast(event.event_type)]) { + if (!trace_event_registry_[event.event_type].is_packet_event()) { continue; } TaskId task_id{event.node_id, event.input_ts, event.event_type}; @@ -195,7 +213,7 @@ class TraceBuilder::Impl { result->set_base_timestamp(base_ts_); std::unordered_set task_ids; for (const TraceEvent& event : snapshot) { - if (!kProfilerPacketEvents[static_cast(event.event_type)]) { + if (!trace_event_registry_[event.event_type].is_packet_event()) { BuildEventLog(event, result->add_calculator_trace()); continue; } @@ -280,18 +298,12 @@ class TraceBuilder::Impl { // Construct the StreamTrace for a TraceEvent. void BuildStreamTrace(const TraceEvent& event, GraphTrace::StreamTrace* result) { - if (event.is_finish) { - result->set_stream_id(stream_id_map_[event.stream_id]); - result->set_packet_timestamp(LogTimestamp(event.packet_ts)); - return; - } result->set_stream_id(stream_id_map_[event.stream_id]); result->set_packet_timestamp(LogTimestamp(event.packet_ts)); - result->set_finish_time(LogTime(event.event_time)); - result->set_packet_id(packet_data_id_map_[event.packet_data_id]); - const TraceEvent* output_event = FindOutputEvent(event); - if (output_event) { - result->set_start_time(LogTime(output_event->event_time)); + if (trace_event_registry_[event.event_type].id_event_data()) { + result->set_event_data(packet_data_id_map_[event.event_data]); + } else { + result->set_event_data(event.event_data); } } @@ -301,10 +313,12 @@ class TraceBuilder::Impl { absl::Time start_time = absl::InfiniteFuture(); absl::Time finish_time = absl::InfiniteFuture(); for (const TraceEvent* event : task_events) { - if (result->input_trace().size() + result->output_trace().size() == 0) { + if (result->event_type() == TraceEvent::UNKNOWN) { result->set_node_id(event->node_id); result->set_event_type(event->event_type); - result->set_input_timestamp(LogTimestamp(event->input_ts)); + if (event->input_ts != Timestamp::Unset()) { + result->set_input_timestamp(LogTimestamp(event->input_ts)); + } result->set_thread_id(event->thread_id); } if (event->is_finish) { @@ -312,12 +326,21 @@ class TraceBuilder::Impl { } else { start_time = std::min(start_time, event->event_time); } - if (kProfilerStreamEvents[static_cast(event->event_type)]) { + if (trace_event_registry_[event->event_type].is_stream_event()) { + auto stream_trace = event->is_finish ? result->add_output_trace() + : result->add_input_trace(); if (event->is_finish) { - BuildStreamTrace(*event, result->add_output_trace()); - auto s = result->output_trace(result->output_trace_size() - 1); + // Log only the packet id for each output event. + stream_trace->set_stream_id(stream_id_map_[event->stream_id]); + stream_trace->set_packet_timestamp(LogTimestamp(event->packet_ts)); } else { - BuildStreamTrace(*event, result->add_input_trace()); + // Log the full stream trace for each input event. + BuildStreamTrace(*event, stream_trace); + stream_trace->set_finish_time(LogTime(event->event_time)); + const TraceEvent* output_event = FindOutputEvent(*event); + if (output_event) { + stream_trace->set_start_time(LogTime(output_event->event_time)); + } } } } @@ -343,13 +366,11 @@ class TraceBuilder::Impl { result->set_input_timestamp(LogTimestamp(event.input_ts)); } result->set_thread_id(event.thread_id); - if (kProfilerStreamEvents[static_cast(event.event_type)]) { + if (trace_event_registry_[event.event_type].is_stream_event()) { if (event.stream_id) { auto stream_trace = event.is_finish ? result->add_output_trace() : result->add_input_trace(); - stream_trace->set_stream_id(stream_id_map_[event.stream_id]); - stream_trace->set_packet_timestamp(LogTimestamp(event.packet_ts)); - stream_trace->set_packet_id(packet_data_id_map_[event.packet_data_id]); + BuildStreamTrace(event, stream_trace); } } } @@ -366,11 +387,17 @@ class TraceBuilder::Impl { int64 base_ts_ = std::numeric_limits::max(); // The time represented as 0 in the trace. int64 base_time_ = std::numeric_limits::max(); + // Indicates traits of each event type. + TraceEventRegistry trace_event_registry_; }; TraceBuilder::TraceBuilder() : impl_(new Impl) {} TraceBuilder::~TraceBuilder() {} +TraceEventRegistry* TraceBuilder::trace_event_registry() { + return impl_->trace_event_registry(); +} + Timestamp TraceBuilder::TimestampAfter(const TraceBuffer& buffer, absl::Time begin_time) { return Impl::TimestampAfter(buffer, begin_time); @@ -400,6 +427,8 @@ const TraceEvent::EventType // TraceEvent::CPU_TASK_SYSTEM, // TraceEvent::GPU_TASK, // TraceEvent::DSP_TASK, // - TraceEvent::TPU_TASK; + TraceEvent::TPU_TASK, // + TraceEvent::GPU_CALIBRATION, // + TraceEvent::PACKET_QUEUED; } // namespace mediapipe diff --git a/mediapipe/framework/profiler/trace_builder.h b/mediapipe/framework/profiler/trace_builder.h index 8c4398984..2bb01c0d3 100644 --- a/mediapipe/framework/profiler/trace_builder.h +++ b/mediapipe/framework/profiler/trace_builder.h @@ -28,6 +28,9 @@ class TraceBuilder { TraceBuilder(); ~TraceBuilder(); + // Returns the registry of trace event types. + TraceEventRegistry* trace_event_registry(); + // Returns the earliest packet timestamp appearing only after begin_time. static Timestamp TimestampAfter(const TraceBuffer& buffer, absl::Time begin_time); diff --git a/mediapipe/framework/stream_handler/default_input_stream_handler.cc b/mediapipe/framework/stream_handler/default_input_stream_handler.cc index a5f6878b1..354ee08e0 100644 --- a/mediapipe/framework/stream_handler/default_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/default_input_stream_handler.cc @@ -45,6 +45,17 @@ DefaultInputStreamHandler::DefaultInputStreamHandler( } } +void DefaultInputStreamHandler::PrepareForRun( + std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) { + sync_set_.PrepareForRun(); + InputStreamHandler::PrepareForRun( + std::move(headers_ready_callback), std::move(notification_callback), + std::move(schedule_callback), std::move(error_callback)); +} + NodeReadiness DefaultInputStreamHandler::GetNodeReadiness( Timestamp* min_stream_timestamp) { return sync_set_.GetReadiness(min_stream_timestamp); diff --git a/mediapipe/framework/stream_handler/default_input_stream_handler.h b/mediapipe/framework/stream_handler/default_input_stream_handler.h index b83dc98b4..0ff9d9a10 100644 --- a/mediapipe/framework/stream_handler/default_input_stream_handler.h +++ b/mediapipe/framework/stream_handler/default_input_stream_handler.h @@ -35,6 +35,13 @@ class DefaultInputStreamHandler : public InputStreamHandler { bool calculator_run_in_parallel); protected: + // Reinitializes this InputStreamHandler before each CalculatorGraph run. + void PrepareForRun( + std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + // In DefaultInputStreamHandler, a node is "ready" if: // - all streams are done (need to call Close() in this case), or // - the minimum bound (over all empty streams) is greater than the smallest diff --git a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc index b34d08498..be21f4180 100644 --- a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc @@ -40,6 +40,13 @@ class ImmediateInputStreamHandler : public InputStreamHandler { const MediaPipeOptions& options, bool calculator_run_in_parallel); protected: + // Reinitializes this InputStreamHandler before each CalculatorGraph run. + void PrepareForRun( + std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + // Returns kReadyForProcess whenever a Packet is available at any of // the input streams, or any input stream becomes done. NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; @@ -69,6 +76,23 @@ ImmediateInputStreamHandler::ImmediateInputStreamHandler( } } +void ImmediateInputStreamHandler::PrepareForRun( + std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) { + { + absl::MutexLock lock(&mutex_); + for (int i = 0; i < sync_sets_.size(); ++i) { + sync_sets_[i].PrepareForRun(); + ready_timestamps_[i] = Timestamp::Unset(); + } + } + InputStreamHandler::PrepareForRun( + std::move(headers_ready_callback), std::move(notification_callback), + std::move(schedule_callback), std::move(error_callback)); +} + NodeReadiness ImmediateInputStreamHandler::GetNodeReadiness( Timestamp* min_stream_timestamp) { absl::MutexLock lock(&mutex_); diff --git a/mediapipe/framework/subgraph.cc b/mediapipe/framework/subgraph.cc index 8d121e01e..1fc69b587 100644 --- a/mediapipe/framework/subgraph.cc +++ b/mediapipe/framework/subgraph.cc @@ -61,6 +61,13 @@ GraphRegistry::GraphRegistry( FunctionRegistry>* factories) : global_factories_(factories) {} +void GraphRegistry::Register( + const std::string& type_name, + std::function()> factory) { + local_factories_.Register(type_name, factory); +} + +// TODO: Remove this convenience function. void GraphRegistry::Register(const std::string& type_name, const CalculatorGraphConfig& config) { local_factories_.Register(type_name, [config] { @@ -69,6 +76,7 @@ void GraphRegistry::Register(const std::string& type_name, }); } +// TODO: Remove this convenience function. void GraphRegistry::Register(const std::string& type_name, const CalculatorGraphTemplate& templ) { local_factories_.Register(type_name, [templ] { diff --git a/mediapipe/framework/subgraph.h b/mediapipe/framework/subgraph.h index 3febde7e9..dce1e536c 100644 --- a/mediapipe/framework/subgraph.h +++ b/mediapipe/framework/subgraph.h @@ -94,6 +94,10 @@ class GraphRegistry { // Ownership of the specified FunctionRegistry is not transferred. GraphRegistry(FunctionRegistry>* factories); + // Registers a graph config builder type, using a factory function. + void Register(const std::string& type_name, + std::function()> factory); + // Registers a graph config by name. void Register(const std::string& type_name, const CalculatorGraphConfig& config); diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 155dca49e..fcff29e08 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -59,7 +59,7 @@ GL_BASE_LINK_OPTS = select({ # runtime. Weak GLESv3 symbols will still be resolved if we # load it early enough. ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "-framework OpenGLES", "-framework CoreVideo", ], @@ -111,7 +111,7 @@ cc_library( # Note: need the frameworks on Apple platforms to get the headers. linkopts = select({ "//conditions:default": [], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "-framework OpenGLES", "-framework CoreVideo", ], @@ -147,7 +147,7 @@ cc_library( "//conditions:default": [ "gl_context_egl.cc", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "gl_context_eagl.cc", ], "//mediapipe:macos": [ @@ -214,7 +214,7 @@ cc_library( "//conditions:default": [ ":gl_texture_buffer", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "//mediapipe/objc:CFHolder", ], "//mediapipe:macos": [ @@ -246,9 +246,14 @@ objc_library( "-Wno-shorten-64-to-32", ], sdk_frameworks = [ + "Accelerate", + "CoreGraphics", "CoreVideo", ], visibility = ["//visibility:public"], + deps = [ + "//mediapipe/objc:util", + ], ) objc_library( @@ -408,7 +413,7 @@ cc_library( "//conditions:default": [ "gl_texture_buffer_pool.cc", ], - "//mediapipe:apple": [], + "//mediapipe:ios": [], "//mediapipe:macos": [ "gl_texture_buffer_pool.cc", ], @@ -417,7 +422,7 @@ cc_library( "//conditions:default": [ "gl_texture_buffer_pool.h", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ # The inclusions check does not see that this is provided by # pixel_buffer_pool_util, so we include it here too. This is # b/28066691. @@ -441,7 +446,7 @@ cc_library( "//conditions:default": [ ":gl_texture_buffer", ], - "//mediapipe:apple": [ + "//mediapipe:ios": [ ":pixel_buffer_pool_util", "//mediapipe/objc:CFHolder", ], diff --git a/mediapipe/gpu/gl_context.cc b/mediapipe/gpu/gl_context.cc index dd1b6fa21..0763e0c3a 100644 --- a/mediapipe/gpu/gl_context.cc +++ b/mediapipe/gpu/gl_context.cc @@ -117,6 +117,7 @@ void* GlContext::DedicatedThread::ThreadBody(void* instance) { void GlContext::DedicatedThread::ThreadBody() { SetThreadName("mediapipe_gl_runner"); + #ifndef __EMSCRIPTEN__ GlThreadCollector::ThreadStarting(); #endif @@ -276,6 +277,11 @@ bool GlContext::HasGlExtension(absl::string_view extension) const { absl::string_view version_string( reinterpret_cast(glGetString(GL_VERSION))); + // We will decide later whether we want to use the version numbers we query + // for, or instead derive that information from the context creation result, + // which we cache here. + GLint gl_major_version_from_context_creation = gl_major_version_; + // Let's try getting the numeric version if possible. glGetIntegerv(GL_MAJOR_VERSION, &gl_major_version_); GLenum err = glGetError(); @@ -293,6 +299,23 @@ bool GlContext::HasGlExtension(absl::string_view extension) const { } } + // If our platform-specific CreateContext already set a major GL version, + // then we use that. Otherwise, we use the queried-for result. We do this + // as a workaround for a Swiftshader on Android bug where the ES2 context + // can report major version 3 instead of 2 when queried. Therefore we trust + // the result from context creation more than from query. See b/152519932 + // for more details. + if (gl_major_version_from_context_creation > 0 && + gl_major_version_ != gl_major_version_from_context_creation) { + LOG(WARNING) << "Requested a context with major GL version " + << gl_major_version_from_context_creation + << " but context reports major version " << gl_major_version_ + << ". Setting to " << gl_major_version_from_context_creation + << ".0"; + gl_major_version_ = gl_major_version_from_context_creation; + gl_minor_version_ = 0; + } + LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_ << " (" << glGetString(GL_VERSION) << ")"; if (gl_major_version_ >= 3) { @@ -613,7 +636,17 @@ std::shared_ptr GlContext::CreateSyncToken() { #if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG token.reset(new GlNopSyncPoint(shared_from_this())); #else - if (SymbolAvailable(&glWaitSync)) { + +#ifdef __EMSCRIPTEN__ + // In Emscripten the glWaitSync function is non-null depending on linkopts, + // but only works in a WebGL2 context, so fall back to use Finish if it is a + // WebGL1/ES2 context. + // TODO: apply this more generally once b/152794517 is fixed. + bool useFenceSync = gl_major_version() > 2; +#else + bool useFenceSync = SymbolAvailable(&glWaitSync); +#endif // __EMSCRIPTEN__ + if (useFenceSync) { token.reset(new GlFenceSyncPoint(shared_from_this())); } else { token.reset(new GlFinishSyncPoint(shared_from_this())); @@ -633,8 +666,30 @@ std::shared_ptr GlContext::TestOnly_CreateSpecificSyncToken( return nullptr; } +// Atomically set var to the greater of its current value or target. +template +static void assign_larger_value(std::atomic* var, T target) { + T current = var->load(); + while (current < target && !var->compare_exchange_weak(current, target)) { + } +} + +// Note: this can get called from an arbitrary thread which is dealing with a +// GlFinishSyncPoint originating from this context. void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { if (gl_finish_count_ > count_to_pass) return; + + // If we've been asked to do a glFinish, note the count we need to reach and + // signal the context our thread may currently be blocked on. + { + absl::MutexLock lock(&mutex_); + assign_larger_value(&gl_finish_count_target_, count_to_pass + 1); + wait_for_gl_finish_cv_.SignalAll(); + if (context_waiting_on_) { + context_waiting_on_->wait_for_gl_finish_cv_.SignalAll(); + } + } + auto finish_task = [this, count_to_pass]() { // When a GlFinishSyncToken is created it takes the current finish count // from the GlContext, and we must wait for gl_finish_count_ to pass it. @@ -646,6 +701,7 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { GlFinishCalled(); } }; + if (IsCurrent()) { // If we are already on the current context, we cannot call // RunWithoutWaiting, since that task will not run until this function @@ -653,13 +709,53 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { finish_task(); return; } + + std::shared_ptr other = GetCurrent(); + if (other) { + // If another context is current, make a note that it is blocked on us, so + // it can signal the right condition variable if it is asked to do a + // glFinish. + absl::MutexLock other_lock(&other->mutex_); + DCHECK(!other->context_waiting_on_); + other->context_waiting_on_ = this; + } // We do not schedule this action using Run because we don't necessarily // want to wait for it to complete. If another job calls GlFinishCalled // sooner, we are done. RunWithoutWaiting(std::move(finish_task)); - absl::MutexLock lock(&mutex_); - while (gl_finish_count_ <= count_to_pass) { - wait_for_gl_finish_cv_.Wait(&mutex_); + { + absl::MutexLock lock(&mutex_); + while (gl_finish_count_ <= count_to_pass) { + if (other && other->gl_finish_count_ < other->gl_finish_count_target_) { + // If another context's dedicated thread is current, it is blocked + // waiting for this context to issue a glFinish call. But this context + // may also block waiting for the other context to do the same: this can + // happen when two contexts are handling each other's GlFinishSyncPoints + // (e.g. a producer and a consumer). To avoid a deadlock a context that + // is waiting on another context must still service Wait calls it may + // receive from its own GlFinishSyncPoints. + // + // We unlock this context's mutex to avoid holding both at the same + // time. + mutex_.Unlock(); + { + glFinish(); + other->GlFinishCalled(); + } + mutex_.Lock(); + // Because we temporarily unlocked mutex_, we cannot wait on the + // condition variable wait away; we need to go back to re-checking the + // condition. Otherwise we might miss a signal. + continue; + } + wait_for_gl_finish_cv_.Wait(&mutex_); + } + } + + if (other) { + // The other context is no longer waiting on us. + absl::MutexLock other_lock(&other->mutex_); + other->context_waiting_on_ = nullptr; } } diff --git a/mediapipe/gpu/gl_context.h b/mediapipe/gpu/gl_context.h index 455b232d7..c3fa1a8fc 100644 --- a/mediapipe/gpu/gl_context.h +++ b/mediapipe/gpu/gl_context.h @@ -380,6 +380,9 @@ class GlContext : public std::enable_shared_from_this { // Changes should be guarded by mutex_. However, we use simple atomic // loads for efficiency on the fast path. std::atomic gl_finish_count_ = ATOMIC_VAR_INIT(0); + std::atomic gl_finish_count_target_ = ATOMIC_VAR_INIT(0); + + GlContext* context_waiting_on_ ABSL_GUARDED_BY(mutex_) = nullptr; // This mutex is held by a thread while this GL context is current on that // thread. Since it may be held for extended periods of time, it should not diff --git a/mediapipe/gpu/gl_texture_buffer_pool.cc b/mediapipe/gpu/gl_texture_buffer_pool.cc index 246b3184a..3a33fb824 100644 --- a/mediapipe/gpu/gl_texture_buffer_pool.cc +++ b/mediapipe/gpu/gl_texture_buffer_pool.cc @@ -26,19 +26,28 @@ GlTextureBufferPool::GlTextureBufferPool(int width, int height, keep_count_(keep_count) {} GlTextureBufferSharedPtr GlTextureBufferPool::GetBuffer() { - absl::MutexLock lock(&mutex_); - std::unique_ptr buffer; - if (available_.empty()) { - buffer = GlTextureBuffer::Create(width_, height_, format_); - if (!buffer) return nullptr; - } else { - buffer = std::move(available_.back()); - available_.pop_back(); - buffer->Reuse(); + bool reuse = false; + + { + absl::MutexLock lock(&mutex_); + if (available_.empty()) { + buffer = GlTextureBuffer::Create(width_, height_, format_); + if (!buffer) return nullptr; + } else { + buffer = std::move(available_.back()); + available_.pop_back(); + reuse = true; + } + + ++in_use_count_; } - ++in_use_count_; + // This needs to wait on consumer sync points, therefore it should not be + // done while holding the mutex. + if (reuse) { + buffer->Reuse(); + } // Return a shared_ptr with a custom deleter that adds the buffer back // to our available list. @@ -60,15 +69,24 @@ std::pair GlTextureBufferPool::GetInUseAndAvailableCounts() { } void GlTextureBufferPool::Return(GlTextureBuffer* buf) { - absl::MutexLock lock(&mutex_); - --in_use_count_; - available_.emplace_back(buf); - TrimAvailable(); + std::vector> trimmed; + { + absl::MutexLock lock(&mutex_); + --in_use_count_; + available_.emplace_back(buf); + TrimAvailable(&trimmed); + } + // The trimmed buffers will be released without holding the lock. } -void GlTextureBufferPool::TrimAvailable() { +void GlTextureBufferPool::TrimAvailable( + std::vector>* trimmed) { int keep = std::max(keep_count_ - in_use_count_, 0); if (available_.size() > keep) { + auto trim_it = std::next(available_.begin(), keep); + if (trimmed) { + std::move(available_.begin(), trim_it, std::back_inserter(*trimmed)); + } available_.resize(keep); } } diff --git a/mediapipe/gpu/gl_texture_buffer_pool.h b/mediapipe/gpu/gl_texture_buffer_pool.h index 783704057..d2e8fc39f 100644 --- a/mediapipe/gpu/gl_texture_buffer_pool.h +++ b/mediapipe/gpu/gl_texture_buffer_pool.h @@ -60,7 +60,8 @@ class GlTextureBufferPool // If the total number of buffers is greater than keep_count, destroys any // surplus buffers that are no longer in use. - void TrimAvailable() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void TrimAvailable(std::vector>* trimmed) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); const int width_; const int height_; diff --git a/mediapipe/gpu/pixel_buffer_pool_util.mm b/mediapipe/gpu/pixel_buffer_pool_util.mm index 5d0906003..1e006fd48 100644 --- a/mediapipe/gpu/pixel_buffer_pool_util.mm +++ b/mediapipe/gpu/pixel_buffer_pool_util.mm @@ -16,6 +16,8 @@ #import +#include "mediapipe/objc/util.h" + #if !defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK) && !defined(NDEBUG) #define ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK 1 #endif // defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK) @@ -27,17 +29,13 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool( CFTimeInterval maxAge) { CVPixelBufferPoolRef pool = NULL; - NSDictionary *sourcePixelBufferOptions = @{ - (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), - (id)kCVPixelBufferWidthKey : @(width), - (id)kCVPixelBufferHeightKey : @(height), -#if TARGET_OS_OSX - (id)kCVPixelFormatOpenGLCompatibility : @(YES), -#else - (id)kCVPixelFormatOpenGLESCompatibility : @(YES), -#endif // TARGET_OS_OSX - (id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ } - }; + NSMutableDictionary *sourcePixelBufferOptions = + [(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy]; + [sourcePixelBufferOptions addEntriesFromDictionary:@{ + (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), + (id)kCVPixelBufferWidthKey : @(width), + (id)kCVPixelBufferHeightKey : @(height), + }]; NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init]; pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount); @@ -131,14 +129,6 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) { CVReturn CreateCVPixelBufferWithoutPool( int width, int height, OSType pixelFormat, CVPixelBufferRef* outBuffer) { - NSDictionary *attributes = @{ -#if TARGET_OS_OSX - (id)kCVPixelFormatOpenGLCompatibility : @(YES), -#else - (id)kCVPixelFormatOpenGLESCompatibility : @(YES), -#endif // TARGET_OS_OSX - (id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ } - }; #if TARGET_IPHONE_SIMULATOR // On the simulator, syncing the texture with the pixelbuffer does not work, // and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not @@ -151,12 +141,12 @@ CVReturn CreateCVPixelBufferWithoutPool( void* data = malloc(bytes_per_row * height); return CVPixelBufferCreateWithBytes( kCFAllocatorDefault, width, height, pixelFormat, data, bytes_per_row, - FreeRefConReleaseCallback, data, (__bridge CFDictionaryRef)attributes, + FreeRefConReleaseCallback, data, GetCVPixelBufferAttributesForGlCompatibility(), outBuffer); #else return CVPixelBufferCreate( kCFAllocatorDefault, width, height, pixelFormat, - (__bridge CFDictionaryRef)attributes, outBuffer); + GetCVPixelBufferAttributesForGlCompatibility(), outBuffer); #endif } diff --git a/mediapipe/java/com/google/mediapipe/components/BUILD b/mediapipe/java/com/google/mediapipe/components/BUILD index f0de669dd..a3eca9b3a 100644 --- a/mediapipe/java/com/google/mediapipe/components/BUILD +++ b/mediapipe/java/com/google/mediapipe/components/BUILD @@ -31,8 +31,8 @@ android_library( "//third_party:androidx_core", "//third_party:androidx_legacy_support_v4", "//third_party:androidx_recyclerview", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) @@ -50,10 +50,10 @@ android_library( "//third_party:androidx_legacy_support_v4", "//third_party:camera2", "//third_party:camerax_core", - "@androidx_concurrent_futures//jar", - "@androidx_lifecycle//jar", - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:androidx_concurrent_concurrent_futures", + "@maven//:androidx_lifecycle_lifecycle_common", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) @@ -67,8 +67,8 @@ android_library( ], visibility = ["//visibility:public"], deps = [ - "@com_google_code_findbugs//jar", - "@com_google_guava_android//jar", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/java/com/google/mediapipe/framework/BUILD b/mediapipe/java/com/google/mediapipe/framework/BUILD index 69520ec97..3e2c7bf6d 100644 --- a/mediapipe/java/com/google/mediapipe/framework/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/BUILD @@ -59,10 +59,10 @@ android_library( ":android_core", "//third_party:androidx_annotation", "//third_party:androidx_legacy_support_v4", - "@com_google_code_findbugs//jar", - "@com_google_common_flogger//jar", - "@com_google_common_flogger_system_backend//jar", - "@com_google_guava_android//jar", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_flogger_flogger", + "@maven//:com_google_flogger_flogger_system_backend", + "@maven//:com_google_guava_guava", ], ) @@ -85,10 +85,10 @@ android_library( "//mediapipe/framework:calculator_java_proto_lite", "//mediapipe/framework:calculator_profile_java_proto_lite", "//mediapipe/framework/tool:calculator_graph_template_java_proto_lite", - "@com_google_code_findbugs//jar", - "@com_google_common_flogger//jar", - "@com_google_common_flogger_system_backend//jar", - "@com_google_guava_android//jar", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_flogger_flogger", + "@maven//:com_google_flogger_flogger_system_backend", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/java/com/google/mediapipe/glutil/BUILD b/mediapipe/java/com/google/mediapipe/glutil/BUILD index 97ebdb187..53ea51046 100644 --- a/mediapipe/java/com/google/mediapipe/glutil/BUILD +++ b/mediapipe/java/com/google/mediapipe/glutil/BUILD @@ -24,10 +24,10 @@ android_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_proguard", - "@com_google_code_findbugs//jar", - "@com_google_common_flogger//jar", - "@com_google_common_flogger_system_backend//jar", - "@com_google_guava_android//jar", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_flogger_flogger", + "@maven//:com_google_flogger_flogger_system_backend", + "@maven//:com_google_guava_guava", ], ) diff --git a/mediapipe/models/README.md b/mediapipe/models/README.md index 1d67d918c..dfa5e4e4d 100644 --- a/mediapipe/models/README.md +++ b/mediapipe/models/README.md @@ -8,17 +8,24 @@ Here are descriptions of the models used in the [example applications](../docs/e ### Face Detection * [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite) - * Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://sites.google.com/corp/view/perception-cv4arvr/blazeface) + * Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://arxiv.org/abs/1907.05047) * [Model card](https://sites.google.com/corp/view/perception-cv4arvr/blazeface#h.p_21ojPZDx3cqq) +### Face Mesh + * [TF.js model](https://tfhub.dev/mediapipe/facemesh/1) + * Paper: ["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724) + * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html) + * [Model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view) + ### Hand Detection and Tracking - * [Palm detection TfLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite) - * [2D hand landmark TfLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite) - * [3D hand landmark TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark_3d.tflite) + * Palm detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite), [TF.js model](https://tfhub.dev/mediapipe/handdetector/1) + * 2D hand landmark: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite) + * 3D hand landmark: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark_3d.tflite), [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1) * [Google AI Blog post](https://mediapipe.page.link/handgoogleaiblog) + * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html) * [Model card](https://mediapipe.page.link/handmc) ### Hair Segmentation * [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite) - * Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation) + * Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://arxiv.org/abs/1907.06740) * [Model card](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation#h.p_NimuO7PgHxlY) diff --git a/mediapipe/objc/BUILD b/mediapipe/objc/BUILD index 00191af1a..8bfff0bf6 100644 --- a/mediapipe/objc/BUILD +++ b/mediapipe/objc/BUILD @@ -19,9 +19,14 @@ cc_library( visibility = ["//mediapipe/framework:mediapipe_internal"], deps = [ ":CFHolder", - "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:packet", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/memory", ], ) diff --git a/mediapipe/objc/MPPGraph.mm b/mediapipe/objc/MPPGraph.mm index 8c9da2011..cccdb945a 100644 --- a/mediapipe/objc/MPPGraph.mm +++ b/mediapipe/objc/MPPGraph.mm @@ -26,6 +26,7 @@ #include "mediapipe/gpu/MPPGraphGPUData.h" #include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_shared_data_internal.h" +#include "mediapipe/objc/util.h" #import "mediapipe/objc/NSError+util_status.h" #import "GTMDefines.h" @@ -116,14 +117,10 @@ void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, if (format == mediapipe::ImageFormat::SRGBA || format == mediapipe::ImageFormat::GRAY8) { CVPixelBufferRef pixelBuffer; - // To ensure compatibility with CVOpenGLESTextureCache, this attribute should be present. - NSDictionary* attributes = @{ - (id)kCVPixelBufferIOSurfacePropertiesKey : @{}, - }; // If kCVPixelFormatType_32RGBA does not work, it returns kCVReturnInvalidPixelFormat. CVReturn error = CVPixelBufferCreate( NULL, frame.Width(), frame.Height(), kCVPixelFormatType_32BGRA, - (__bridge CFDictionaryRef)attributes, &pixelBuffer); + GetCVPixelBufferAttributesForGlCompatibility(), &pixelBuffer); _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferCreate failed: %d", error); error = CVPixelBufferLockBaseAddress(pixelBuffer, 0); _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferLockBaseAddress failed: %d", error); diff --git a/mediapipe/objc/util.cc b/mediapipe/objc/util.cc index 756cf171d..d09b40fc4 100644 --- a/mediapipe/objc/util.cc +++ b/mediapipe/objc/util.cc @@ -517,17 +517,29 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() { CFDictionaryRef empty_dict = CFDictionaryCreate( kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + // To ensure compatibility with CVOpenGLESTextureCache, these attributes - // should be present. + // should be present. However, on simulator this IOSurface attribute + // actually causes CVOpenGLESTextureCache to fail. b/144850076 const void* keys[] = { +#if !TARGET_IPHONE_SIMULATOR kCVPixelBufferIOSurfacePropertiesKey, +#endif // !TARGET_IPHONE_SIMULATOR + #if TARGET_OS_OSX kCVPixelFormatOpenGLCompatibility, #else kCVPixelFormatOpenGLESCompatibility, #endif // TARGET_OS_OSX }; - const void* values[] = {empty_dict, kCFBooleanTrue}; + + const void* values[] = { +#if !TARGET_IPHONE_SIMULATOR + empty_dict, +#endif // !TARGET_IPHONE_SIMULATOR + kCFBooleanTrue + }; + attrs = CFDictionaryCreate( kCFAllocatorDefault, keys, values, ABSL_ARRAYSIZE(values), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index c7a4ba95b..ffa7ed1ca 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -173,14 +173,14 @@ cc_library( srcs = select({ "//conditions:default": ["resource_util.cc"], "//mediapipe:android": ["resource_util_android.cc"], - "//mediapipe:apple": ["resource_util_apple.cc"], + "//mediapipe:ios": ["resource_util_apple.cc"], "//mediapipe:macos": ["resource_util.cc"], }), hdrs = ["resource_util.h"], # We use Objective-C++ on iOS. copts = select({ "//conditions:default": [], - "//mediapipe:apple": [ + "//mediapipe:ios": [ "-ObjC++", ], "//mediapipe:macos": [], @@ -201,7 +201,7 @@ cc_library( "//mediapipe/util/android:asset_manager_util", "//mediapipe/util/android/file/base", ], - "//mediapipe:apple": [], + "//mediapipe:ios": [], "//mediapipe:macos": [ "//mediapipe/framework/port:file_helpers", ], diff --git a/mediapipe/util/sequence/media_sequence.h b/mediapipe/util/sequence/media_sequence.h index cd48bfe1e..51934a509 100644 --- a/mediapipe/util/sequence/media_sequence.h +++ b/mediapipe/util/sequence/media_sequence.h @@ -404,6 +404,15 @@ void ClearPoint(const std::string& prefix, kRegionClassIndexKey, prefix) \ FIXED_PREFIX_VECTOR_INT64_FEATURE_LIST(CONCAT_STR2(identifier, TrackIndex), \ kRegionTrackIndexKey, prefix) \ + FIXED_PREFIX_VECTOR_FLOAT_FEATURE_LIST( \ + CONCAT_STR2(identifier, LabelConfidence), kRegionLabelConfidenceKey, \ + prefix) \ + FIXED_PREFIX_VECTOR_FLOAT_FEATURE_LIST( \ + CONCAT_STR2(identifier, ClassConfidence), kRegionClassConfidenceKey, \ + prefix) \ + FIXED_PREFIX_VECTOR_FLOAT_FEATURE_LIST( \ + CONCAT_STR2(identifier, TrackConfidence), kRegionTrackConfidenceKey, \ + prefix) \ FIXED_PREFIX_VECTOR_INT64_FEATURE_LIST(CONCAT_STR2(identifier, IsGenerated), \ kRegionIsGeneratedKey, prefix) \ FIXED_PREFIX_VECTOR_INT64_FEATURE_LIST(CONCAT_STR2(identifier, IsOccluded), \ diff --git a/mediapipe/util/sequence/media_sequence_test.cc b/mediapipe/util/sequence/media_sequence_test.cc index 93b3267bd..0b860d8ea 100644 --- a/mediapipe/util/sequence/media_sequence_test.cc +++ b/mediapipe/util/sequence/media_sequence_test.cc @@ -352,6 +352,19 @@ TEST(MediaSequenceTest, RoundTripBBoxTrackString) { } } +TEST(MediaSequenceTest, RoundTripBBoxTrackConfidence) { + tensorflow::SequenceExample sequence; + std::vector> confidences = {{0.5, 0.3}, {0.1, 0.2}}; + for (int i = 0; i < confidences.size(); ++i) { + AddBBoxTrackConfidence(confidences[i], &sequence); + ASSERT_EQ(GetBBoxTrackConfidenceSize(sequence), i + 1); + const auto& sequence_confidences = GetBBoxTrackConfidenceAt(sequence, i); + for (int j = 0; j < sequence_confidences.size(); ++j) { + ASSERT_EQ(sequence_confidences[j], confidences[i][j]); + } + } +} + TEST(MediaSequenceTest, RoundTripBBoxTimestamp) { tensorflow::SequenceExample sequence; std::vector timestamps = {5, 3}; @@ -409,6 +422,20 @@ TEST(MediaSequenceTest, RoundTripBBoxPoint) { } } +TEST(MediaSequenceTest, RoundTripBBoxPointPrefixed) { + tensorflow::SequenceExample sequence; + std::vector>> points = { + {{0.3, 0.5}, {0.4, 0.7}}, {{0.7, 0.5}, {0.3, 0.4}}}; + for (int i = 0; i < points.size(); ++i) { + AddBBoxPoint("TEST", points[i], &sequence); + ASSERT_EQ(GetBBoxPointSize("TEST", sequence), i + 1); + const auto& sequence_points = GetBBoxPointAt("TEST", sequence, i); + for (int j = 0; j < sequence_points.size(); ++j) { + EXPECT_EQ(sequence_points[j], points[i][j]); + } + } +} + TEST(MediaSequenceTest, RoundTripRegionParts) { tensorflow::SequenceExample sequence; std::vector parts = {"HEAD", "FEET"}; diff --git a/mediapipe/util/tracking/BUILD b/mediapipe/util/tracking/BUILD index 5bc125c34..7e648c95e 100644 --- a/mediapipe/util/tracking/BUILD +++ b/mediapipe/util/tracking/BUILD @@ -117,6 +117,20 @@ proto_library( deps = [":tracking_proto"], ) +proto_library( + name = "tracked_detection_manager_config_proto", + srcs = ["tracked_detection_manager_config.proto"], +) + +proto_library( + name = "box_detector_proto", + srcs = ["box_detector.proto"], + deps = [ + ":box_tracker_proto", + ":region_flow_proto", + ], +) + mediapipe_cc_proto_library( name = "tone_models_cc_proto", srcs = ["tone_models.proto"], @@ -245,6 +259,24 @@ mediapipe_cc_proto_library( deps = [":box_tracker_proto"], ) +mediapipe_cc_proto_library( + name = "tracked_detection_manager_config_cc_proto", + srcs = ["tracked_detection_manager_config.proto"], + visibility = ["//visibility:public"], + deps = [":tracked_detection_manager_config_proto"], +) + +mediapipe_cc_proto_library( + name = "box_detector_cc_proto", + srcs = ["box_detector.proto"], + cc_deps = [ + ":box_tracker_cc_proto", + ":region_flow_cc_proto", + ], + visibility = ["//visibility:public"], + deps = [":box_detector_proto"], +) + cc_library( name = "motion_models", srcs = ["motion_models.cc"], @@ -612,6 +644,28 @@ cc_library( ], ) +cc_library( + name = "box_detector", + srcs = ["box_detector.cc"], + hdrs = ["box_detector.h"], + deps = [ + ":box_detector_cc_proto", + ":box_tracker", + ":box_tracker_cc_proto", + ":flow_packager_cc_proto", + ":measure_time", + ":tracking", + "//mediapipe/framework/port:opencv_calib3d", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_features2d", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/synchronization", + ], +) + cc_library( name = "tracking_visualization_utilities", srcs = ["tracking_visualization_utilities.cc"], @@ -737,6 +791,7 @@ cc_library( ], deps = [ ":tracked_detection", + ":tracked_detection_manager_config_cc_proto", "//mediapipe/framework/formats:rect_cc_proto", "@com_google_absl//absl/container:node_hash_map", ], diff --git a/mediapipe/util/tracking/box_detector.cc b/mediapipe/util/tracking/box_detector.cc new file mode 100644 index 000000000..02ac6321a --- /dev/null +++ b/mediapipe/util/tracking/box_detector.cc @@ -0,0 +1,749 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/util/tracking/box_detector.h" + +#include + +#include "absl/memory/memory.h" +#include "mediapipe/framework/port/opencv_calib3d_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/util/tracking/box_detector.pb.h" +#include "mediapipe/util/tracking/box_tracker.h" +#include "mediapipe/util/tracking/measure_time.h" + +namespace mediapipe { + +namespace { + +void ScaleBox(float scale_x, float scale_y, TimedBoxProto *box) { + box->set_left(box->left() * scale_x); + box->set_right(box->right() * scale_x); + box->set_top(box->top() * scale_y); + box->set_bottom(box->bottom() * scale_y); + + if (box->has_quad()) { + for (int c = 0; c < 4; ++c) { + (*box->mutable_quad()->mutable_vertices())[2 * c] *= scale_x; + (*box->mutable_quad()->mutable_vertices())[2 * c + 1] *= scale_y; + } + } +} + +cv::Mat ConvertDescriptorsToMat(const std::vector &descriptors) { + CHECK(!descriptors.empty()) << "empty descriptors."; + + const int descriptors_dims = descriptors[0].size(); + CHECK_GT(descriptors_dims, 0); + + cv::Mat mat(descriptors.size(), descriptors_dims, CV_8U); + + for (int j = 0; j < descriptors.size(); ++j) { + memcpy(mat.row(j).data, descriptors[j].data(), descriptors_dims); + } + + return mat; +} + +cv::Mat GetDescriptorsWithIndices(const cv::Mat &frame_descriptors, + const std::vector &indices) { + CHECK_GT(frame_descriptors.rows, 0); + + const int num_inlier_descriptors = indices.size(); + CHECK_GT(num_inlier_descriptors, 0); + + const int descriptors_dims = frame_descriptors.cols; + CHECK_GT(descriptors_dims, 0); + + cv::Mat mat(num_inlier_descriptors, descriptors_dims, CV_32F); + + for (int j = 0; j < indices.size(); ++j) { + frame_descriptors.row(indices[j]).copyTo(mat.row(j)); + } + + return mat; +} + +} // namespace + +// Using OpenCV brute force matcher along with cross validate match to conduct +// the query. +class BoxDetectorOpencvBfImpl : public BoxDetectorInterface { + public: + explicit BoxDetectorOpencvBfImpl(const BoxDetectorOptions &options); + + private: + std::vector MatchFeatureDescriptors( + const std::vector &features, const cv::Mat &descriptors, + int box_idx) override; + + cv::BFMatcher bf_matcher_; +}; + +std::unique_ptr BoxDetectorInterface::Create( + const BoxDetectorOptions &options) { + if (options.index_type() == BoxDetectorOptions::OPENCV_BF) { + return absl::make_unique(options); + } else { + LOG(FATAL) << "index type undefined."; + } +} + +BoxDetectorInterface::BoxDetectorInterface(const BoxDetectorOptions &options) + : options_(options) {} + +void BoxDetectorInterface::DetectAndAddBoxFromFeatures( + const std::vector &features, const cv::Mat &descriptors, + const TimedBoxProtoList &tracked_boxes, int64 timestamp_msec, float scale_x, + float scale_y, TimedBoxProtoList *detected_boxes) { + absl::MutexLock lock_access(&access_to_index_); + image_scale_ = std::min(scale_x, scale_y); + image_aspect_ = scale_x / scale_y; + + int size_before_add = box_id_to_idx_.size(); + std::vector tracked(size_before_add, false); + for (const auto &box : tracked_boxes.box()) { + if (!box.reacquisition()) { + continue; + } + + const absl::flat_hash_map::iterator iter = + box_id_to_idx_.find(box.id()); + if (iter == box_id_to_idx_.end()) { + // De-normalize the input box to image scale + TimedBoxProto scaled_box = box; + ScaleBox(scale_x, scale_y, &scaled_box); + + AddBoxFeaturesToIndex(features, descriptors, scaled_box, + /*transform_features_for_pnp*/ true); + } else { + int box_idx = iter->second; + tracked[box_idx] = true; + + float center_x = 0.0f; + float center_y = 0.0f; + if (!box.has_quad()) { + center_x = (box.left() + box.right()) * 0.5f; + center_y = (box.top() + box.bottom()) * 0.5f; + } else { + for (int c = 0; c < 4; ++c) { + center_x += box.quad().vertices(c * 2); + center_y += box.quad().vertices(c * 2 + 1); + } + center_x /= 4; + center_y /= 4; + } + + if (center_x < 0.0f || center_x > 1.0f || center_y < 0.0f || + center_y > 1.0f) { + has_been_out_of_fov_[box_idx] = true; + } + } + } + + for (int idx = 0; idx < size_before_add; ++idx) { + if ((options_.has_detect_every_n_frame() > 0 && + cnt_detect_called_ % options_.detect_every_n_frame() == 0) || + !tracked[idx] || + (options_.detect_out_of_fov() && has_been_out_of_fov_[idx])) { + TimedBoxProtoList det = DetectBox(features, descriptors, idx); + if (det.box_size() > 0) { + det.mutable_box(0)->set_time_msec(timestamp_msec); + + // Convert the result box to normalized space. + ScaleBox(1.0f / scale_x, 1.0f / scale_y, det.mutable_box(0)); + *detected_boxes->add_box() = det.box(0); + + has_been_out_of_fov_[idx] = false; + } + } + } + + // reset timer after detect or add action. + cnt_detect_called_ = 1; +} + +void BoxDetectorInterface::DetectAndAddBox( + const TrackingData &tracking_data, const TimedBoxProtoList &tracked_boxes, + int64 timestamp_msec, TimedBoxProtoList *detected_boxes) { + std::vector features_from_tracking_data; + std::vector descriptors_from_tracking_data; + FeatureAndDescriptorFromTrackingData(tracking_data, + &features_from_tracking_data, + &descriptors_from_tracking_data); + + if (features_from_tracking_data.empty() || + descriptors_from_tracking_data.empty()) { + LOG(WARNING) << "Detection skipped due to empty features or descriptors."; + return; + } + + cv::Mat frame_descriptors = + ConvertDescriptorsToMat(descriptors_from_tracking_data); + + float scale_x, scale_y; + ScaleFromAspect(tracking_data.frame_aspect(), /*invert*/ false, &scale_x, + &scale_y); + + DetectAndAddBoxFromFeatures(features_from_tracking_data, frame_descriptors, + tracked_boxes, timestamp_msec, scale_x, scale_y, + detected_boxes); +} + +bool BoxDetectorInterface::CheckDetectAndAddBox( + const TimedBoxProtoList &tracked_boxes) { + bool need_add = false; + int cnt_tracked = 0; + for (const auto &box : tracked_boxes.box()) { + if (!box.reacquisition()) { + continue; + } + + const absl::flat_hash_map::iterator iter = + box_id_to_idx_.find(box.id()); + if (iter == box_id_to_idx_.end()) { + need_add = true; + break; + } else { + ++cnt_tracked; + } + } + // When new boxes being added for reacquisition, we need to run redetection. + if (need_add) { + return true; + } + + const bool is_periodical_check_on = options_.detect_every_n_frame() > 0; + const bool need_periodical_check = + is_periodical_check_on && + (cnt_detect_called_ % options_.detect_every_n_frame() == 0); + + // When configured to do periodical check, and when need to run the periodical + // check, we run redetection. + if (need_periodical_check) { + return true; + } + + const bool any_reacquisition_box_missing = + !box_id_to_idx_.empty() && (cnt_tracked < box_id_to_idx_.size()); + + // When NOT configured to use periodical check, we run redetection EVERY frame + // when any reacquisition box is missing. Note this path of redetection + // including re-run feature extraction and is expensive, might cause graph + // throttling on low end devices. + if (!is_periodical_check_on && any_reacquisition_box_missing) { + return true; + } + + // Other cases, increment the cnt_detect_called_ number and return false to + // not run redetection. + ++cnt_detect_called_; + return false; +} + +void BoxDetectorInterface::DetectAndAddBox( + const cv::Mat &image, const TimedBoxProtoList &tracked_boxes, + int64 timestamp_msec, TimedBoxProtoList *detected_boxes) { + // Determine if we need execute feature extraction. + if (!CheckDetectAndAddBox(tracked_boxes)) { + return; + } + + const auto &image_query_settings = options_.image_query_settings(); + + cv::Mat grayscale; + if (image.channels() == 3) { + cv::cvtColor(image, grayscale, cv::COLOR_BGR2GRAY); + } else if (image.channels() == 4) { + cv::cvtColor(image, grayscale, cv::COLOR_RGBA2GRAY); + } else { + grayscale = image; + } + + cv::Mat resize_image; + const int longer_edge = std::max(grayscale.cols, grayscale.rows); + const float longer_edge_scaled = image_query_settings.pyramid_bottom_size(); + if (longer_edge <= longer_edge_scaled) { + resize_image = grayscale; + } else { + float resize_scale = longer_edge_scaled / longer_edge; + cv::resize( + grayscale, resize_image, + cv::Size(resize_scale * grayscale.cols, resize_scale * grayscale.rows), + cv::INTER_AREA); + } + + // Use cv::ORB feature extractor for now since it provides better quality of + // detection results compared with manually constructing pyramid and then use + // OrbFeatureDescriptor. + // TODO: Tune OrbFeatureDescriptor to hit similar quality. + if (!orb_extractor_) { + orb_extractor_ = + cv::ORB::create(image_query_settings.max_features(), + image_query_settings.pyramid_scale_factor(), + image_query_settings.max_pyramid_levels()); + } + + std::vector keypoints; + cv::Mat descriptors; + orb_extractor_->detect(resize_image, keypoints); + orb_extractor_->compute(resize_image, keypoints, descriptors); + + CHECK_EQ(keypoints.size(), descriptors.rows); + + float inv_scale = 1.0f / std::max(resize_image.cols, resize_image.rows); + std::vector v_keypoints(keypoints.size()); + for (int j = 0; j < keypoints.size(); ++j) { + v_keypoints[j] = + Vector2_f(keypoints[j].pt.x * inv_scale, keypoints[j].pt.y * inv_scale); + } + + float scale_x = resize_image.cols * inv_scale; + float scale_y = resize_image.rows * inv_scale; + + DetectAndAddBoxFromFeatures(v_keypoints, descriptors, tracked_boxes, + timestamp_msec, scale_x, scale_y, detected_boxes); +} + +TimedBoxProtoList BoxDetectorInterface::DetectBox( + const std::vector &features, const cv::Mat &descriptors, + int box_idx) { + return FindBoxesFromFeatureCorrespondence( + MatchFeatureDescriptors(features, descriptors, box_idx), box_idx); +} + +TimedBoxProtoList BoxDetectorInterface::FindBoxesFromFeatureCorrespondence( + const std::vector &matches, int box_idx) { + int max_corr = -1; + int max_corr_frame = 0; + for (int j = 0; j < matches.size(); ++j) { + int num_corr = matches[j].points_frame.size(); + if (num_corr > max_corr) { + max_corr = num_corr; + max_corr_frame = j; + } + } + + TimedBoxProtoList result_list; + + constexpr int kMinNumCorrespondence = 10; + if (max_corr < kMinNumCorrespondence) { + return result_list; + } + + const TimedBoxProto &ori_box = frame_box_[box_idx][max_corr_frame]; + if (!ori_box.has_quad()) { + cv::Mat similarity = + cv::estimateRigidTransform(matches[max_corr_frame].points_index, + matches[max_corr_frame].points_frame, false); + + if (similarity.cols == 0 || similarity.rows == 0) return result_list; + + float similarity_scale = + std::hypot(similarity.at(0, 0), similarity.at(1, 0)); + float similarity_theta = + std::atan2(similarity.at(1, 0), similarity.at(0, 0)); + + auto *new_box_ptr = result_list.add_box(); + + float box_center_x = 0.5f * (ori_box.left() + ori_box.right()); + float box_center_y = 0.5f * (ori_box.top() + ori_box.bottom()); + + float new_center_x = similarity.at(0, 0) * box_center_x + + similarity.at(0, 1) * box_center_y + + similarity.at(0, 2); + float new_center_y = similarity.at(1, 0) * box_center_x + + similarity.at(1, 1) * box_center_y + + similarity.at(1, 2); + + new_box_ptr->set_left((ori_box.left() - box_center_x) * similarity_scale + + new_center_x); + new_box_ptr->set_right((ori_box.right() - box_center_x) * similarity_scale + + new_center_x); + + new_box_ptr->set_top((ori_box.top() - box_center_y) * similarity_scale + + new_center_y); + new_box_ptr->set_bottom( + (ori_box.bottom() - box_center_y) * similarity_scale + new_center_y); + + new_box_ptr->set_rotation(ori_box.rotation() + similarity_theta); + + new_box_ptr->set_id(box_idx_to_id_[box_idx]); + new_box_ptr->set_reacquisition(true); + return result_list; + } else { + return FindQuadFromFeatureCorrespondence(matches[max_corr_frame], ori_box, + image_aspect_); + } +} + +TimedBoxProtoList BoxDetectorInterface::FindQuadFromFeatureCorrespondence( + const FeatureCorrespondence &matches, const TimedBoxProto &box_proto, + float frame_aspect) { + TimedBoxProtoList result_list; + + if (matches.points_frame.size() != matches.points_index.size()) { + LOG(ERROR) << matches.points_frame.size() << " vs " + << matches.points_index.size() + << ". Correpondence size doesn't match."; + return result_list; + } + + int matches_size = matches.points_frame.size(); + if (matches_size < options_.min_num_correspondence()) { + return result_list; + } + + constexpr int kRansacMaxIterations = 100; + cv::Mat inliers_set; + cv::Mat homography = cv::findHomography( + matches.points_index, matches.points_frame, cv::FM_RANSAC, + options_.ransac_reprojection_threshold(), inliers_set, + kRansacMaxIterations); + + // Check if the orientation is preserved, otherwise quad will be flipped. + double det = homography.at(0, 0) * homography.at(1, 1) - + homography.at(0, 1) * homography.at(1, 0); + if (det < 0) { + return result_list; + } + + double persp = homography.at(2, 0) * homography.at(2, 0) + + homography.at(2, 1) * homography.at(2, 1); + if (persp > options_.max_perspective_factor()) { + return result_list; + } + + std::vector frame_corners; + + if (frame_aspect > 0.0f && box_proto.has_aspect_ratio() && + box_proto.aspect_ratio() > 0.0f) { + float box_scale_x, box_scale_y; + ScaleFromAspect(box_proto.aspect_ratio(), /*invert*/ false, &box_scale_x, + &box_scale_y); + const float box_half_x = box_scale_x * 0.5; + const float box_half_y = box_scale_y * 0.5; + + float frame_scale_x, frame_scale_y; + ScaleFromAspect(frame_aspect, /*invert*/ false, &frame_scale_x, + &frame_scale_y); + const float frame_half_x = frame_scale_x * 0.5; + const float frame_half_y = frame_scale_y * 0.5; + + std::vector vectors_3d; + vectors_3d.reserve(matches_size); + std::vector vectors_2d; + vectors_2d.reserve(matches_size); + + for (int j = 0; j < matches_size; ++j) { + if (inliers_set.at(j)) { + vectors_3d.emplace_back(matches.points_index[j].x - box_half_x, + matches.points_index[j].y - box_half_y, 0.0f); + vectors_2d.emplace_back(matches.points_frame[j].x - frame_half_x, + matches.points_frame[j].y - frame_half_y); + } + } + + constexpr int kMinCorrespondences = 4; + if (vectors_3d.size() < kMinCorrespondences) { + return result_list; + } + + // TODO: Use camera intrinsic if provided. + cv::Mat rvec, tvec; + cv::solvePnP(vectors_3d, vectors_2d, cv::Mat::eye(3, 3, CV_64F), + cv::Mat::zeros(1, 5, CV_64FC1), rvec, tvec); + + std::vector template_corners{ + cv::Point3f(-box_half_x, -box_half_y, 0), + cv::Point3f(-box_half_x, box_half_y, 0), + cv::Point3f(box_half_x, box_half_y, 0), + cv::Point3f(box_half_x, -box_half_y, 0)}; + + cv::projectPoints(template_corners, rvec, tvec, cv::Mat::eye(3, 3, CV_64F), + cv::Mat::zeros(1, 5, CV_64FC1), frame_corners); + + for (int j = 0; j < 4; ++j) { + frame_corners[j].x += frame_half_x; + frame_corners[j].y += frame_half_y; + } + } else { + std::vector template_corners{ + cv::Point2f(box_proto.quad().vertices(0), box_proto.quad().vertices(1)), + cv::Point2f(box_proto.quad().vertices(2), box_proto.quad().vertices(3)), + cv::Point2f(box_proto.quad().vertices(4), box_proto.quad().vertices(5)), + cv::Point2f(box_proto.quad().vertices(6), + box_proto.quad().vertices(7))}; + + cv::perspectiveTransform(template_corners, frame_corners, homography); + } + + auto *new_box_ptr = result_list.add_box(); + + float min_x = std::numeric_limits::max(); + float max_x = std::numeric_limits::lowest(); + float min_y = std::numeric_limits::max(); + float max_y = std::numeric_limits::lowest(); + for (int c = 0; c < 4; ++c) { + new_box_ptr->mutable_quad()->add_vertices(frame_corners[c].x); + new_box_ptr->mutable_quad()->add_vertices(frame_corners[c].y); + + min_x = std::min(min_x, frame_corners[c].x); + max_x = std::max(max_x, frame_corners[c].x); + min_y = std::min(min_y, frame_corners[c].y); + max_y = std::max(max_y, frame_corners[c].y); + } + + new_box_ptr->set_left(min_x); + new_box_ptr->set_right(max_x); + new_box_ptr->set_top(min_y); + new_box_ptr->set_bottom(max_y); + new_box_ptr->set_rotation(0.0f); + new_box_ptr->set_id(box_proto.id()); + new_box_ptr->set_reacquisition(true); + if (box_proto.has_aspect_ratio()) { + new_box_ptr->set_aspect_ratio(box_proto.aspect_ratio()); + } + + return result_list; +} + +std::vector BoxDetectorInterface::GetFeatureIndexWithinBox( + const std::vector &features, const TimedBoxProto &box) { + std::vector insider_idx; + if (features.empty()) return insider_idx; + + MotionBoxState box_state; + if (!box.has_quad()) { + box_state.set_pos_x(box.left()); + box_state.set_pos_y(box.top()); + box_state.set_width(box.right() - box.left()); + box_state.set_height(box.bottom() - box.top()); + box_state.set_rotation(box.rotation()); + } else { + auto *state_quad_ptr = box_state.mutable_quad(); + for (int c = 0; c < 8; ++c) { + state_quad_ptr->add_vertices(box.quad().vertices(c)); + } + } + + const Vector2_f box_scaling(1.0f, 1.0f); + constexpr float kScaleFactorForBoxEnlarging = 0.1f; + constexpr int kMinNumFeatures = 60; + GetFeatureIndicesWithinBox( + features, box_state, box_scaling, + /*max_enlarge_size=*/image_scale_ * kScaleFactorForBoxEnlarging, + /*min_num_features=*/kMinNumFeatures, &insider_idx); + return insider_idx; +} + +void BoxDetectorInterface::AddBoxFeaturesToIndex( + const std::vector &features, const cv::Mat &descriptors, + const TimedBoxProto &box, bool transform_features_for_pnp) { + std::vector insider_idx = GetFeatureIndexWithinBox(features, box); + + if (!insider_idx.empty()) { + const absl::flat_hash_map::iterator iter = + box_id_to_idx_.find(box.id()); + int box_idx; + if (iter == box_id_to_idx_.end()) { + box_idx = box_id_to_idx_.size(); + box_id_to_idx_[box.id()] = box_idx; + box_idx_to_id_.push_back(box.id()); + frame_box_.resize(box_id_to_idx_.size()); + feature_to_frame_.resize(box_id_to_idx_.size()); + feature_keypoints_.resize(box_id_to_idx_.size()); + feature_descriptors_.resize(box_id_to_idx_.size()); + has_been_out_of_fov_.push_back(false); + } else { + box_idx = iter->second; + has_been_out_of_fov_[box_idx] = false; + } + + // Create a frame + int frame_id = frame_box_[box_idx].size(); + frame_box_[box_idx].push_back(box); + + cv::Mat box_descriptors = + GetDescriptorsWithIndices(descriptors, insider_idx); + if (feature_descriptors_[box_idx].rows == 0) { + feature_descriptors_[box_idx] = box_descriptors; + } else { + cv::vconcat(feature_descriptors_[box_idx], box_descriptors, + feature_descriptors_[box_idx]); + } + + if (box.has_aspect_ratio() && transform_features_for_pnp) { + // TODO: Dynamically switching between pnp and homography + // detection is not supported. The detector can only perform detection in + // one mode in its lifetime. + float scale_x, scale_y; + ScaleFromAspect(box.aspect_ratio(), /*invert*/ false, &scale_x, &scale_y); + std::vector corners_template{ + cv::Point2f(0.0f, 0.0f), cv::Point2f(0.0f, scale_y), + cv::Point2f(scale_x, scale_y), cv::Point2f(scale_x, 0.0f)}; + std::vector corners_frame(4); + for (int j = 0; j < 4; ++j) { + corners_frame[j].x = box.quad().vertices(j * 2); + corners_frame[j].y = box.quad().vertices(j * 2 + 1); + } + cv::Mat h_transform = cv::findHomography(corners_frame, corners_template); + std::vector features_frame, features_template; + for (int j = 0; j < insider_idx.size(); ++j) { + features_frame.emplace_back(features[insider_idx[j]].x(), + features[insider_idx[j]].y()); + } + cv::perspectiveTransform(features_frame, features_template, h_transform); + for (int j = 0; j < features_template.size(); ++j) { + feature_keypoints_[box_idx].emplace_back(features_template[j].x, + features_template[j].y); + } + } else { + for (int j = 0; j < insider_idx.size(); ++j) { + feature_keypoints_[box_idx].emplace_back(features[insider_idx[j]]); + } + } + + for (int j = 0; j < insider_idx.size(); ++j) { + feature_to_frame_[box_idx].push_back(frame_id); + } + } +} + +void BoxDetectorInterface::CancelBoxDetection(int box_id) { + absl::MutexLock lock_access(&access_to_index_); + const absl::flat_hash_map::iterator iter = + box_id_to_idx_.find(box_id); + if (iter == box_id_to_idx_.end()) { + return; + } else { + const int erase_idx = iter->second; + frame_box_.erase(frame_box_.begin() + erase_idx); + feature_to_frame_.erase(feature_to_frame_.begin() + erase_idx); + feature_keypoints_.erase(feature_keypoints_.begin() + erase_idx); + feature_descriptors_.erase(feature_descriptors_.begin() + erase_idx); + has_been_out_of_fov_.erase(has_been_out_of_fov_.begin() + erase_idx); + box_idx_to_id_.erase(box_idx_to_id_.begin() + erase_idx); + box_id_to_idx_.erase(iter); + for (int j = erase_idx; j < box_idx_to_id_.size(); ++j) { + box_id_to_idx_[box_idx_to_id_[j]] = j; + } + } +} + +BoxDetectorIndex BoxDetectorInterface::ObtainBoxDetectorIndex() const { + absl::MutexLock lock_access(&access_to_index_); + BoxDetectorIndex index; + for (int j = 0; j < frame_box_.size(); ++j) { + BoxDetectorIndex::BoxEntry *box_ptr = index.add_box_entry(); + for (int i = 0; i < frame_box_[j].size(); ++i) { + BoxDetectorIndex::BoxEntry::FrameEntry *frame_ptr = + box_ptr->add_frame_entry(); + *(frame_ptr->mutable_box()) = frame_box_[j][i]; + } + + for (int k = 0; k < feature_to_frame_[j].size(); ++k) { + BoxDetectorIndex::BoxEntry::FrameEntry *frame_ptr = + box_ptr->mutable_frame_entry(feature_to_frame_[j][k]); + + frame_ptr->add_keypoints(feature_keypoints_[j][k].x()); + frame_ptr->add_keypoints(feature_keypoints_[j][k].y()); + frame_ptr->add_descriptors()->set_data( + static_cast(feature_descriptors_[j].row(k).data), + feature_descriptors_[j].cols * sizeof(float)); + } + } + + return index; +} + +void BoxDetectorInterface::AddBoxDetectorIndex(const BoxDetectorIndex &index) { + absl::MutexLock lock_access(&access_to_index_); + for (int j = 0; j < index.box_entry_size(); ++j) { + const auto &box_entry = index.box_entry(j); + for (int i = 0; i < box_entry.frame_entry_size(); ++i) { + const auto &frame_entry = box_entry.frame_entry(i); + + // If the box to be added already exists in the index, skip. + if (box_id_to_idx_.find(frame_entry.box().id()) != box_id_to_idx_.end()) { + continue; + } + + CHECK_EQ(frame_entry.keypoints_size(), + frame_entry.descriptors_size() * 2); + + const int num_features = frame_entry.descriptors_size(); + CHECK_GT(num_features, 0); + std::vector features(num_features); + + const int descriptors_dims = frame_entry.descriptors(0).data().size(); + CHECK_GT(descriptors_dims, 0); + + cv::Mat descriptors_mat(num_features, descriptors_dims / sizeof(float), + CV_32F); + for (int k = 0; k < num_features; ++k) { + features[k] = Vector2_f(frame_entry.keypoints(2 * k), + frame_entry.keypoints(2 * k + 1)); + memcpy(descriptors_mat.row(k).data, + frame_entry.descriptors(k).data().data(), descriptors_dims); + } + + AddBoxFeaturesToIndex(features, descriptors_mat, frame_entry.box()); + } + } +} + +BoxDetectorOpencvBfImpl::BoxDetectorOpencvBfImpl( + const BoxDetectorOptions &options) + : BoxDetectorInterface(options), bf_matcher_(cv::NORM_L2, true) {} + +std::vector +BoxDetectorOpencvBfImpl::MatchFeatureDescriptors( + const std::vector &features, const cv::Mat &descriptors, + int box_idx) { + CHECK_EQ(features.size(), descriptors.rows); + + std::vector correspondence_result( + frame_box_[box_idx].size()); + if (features.empty() || descriptors.rows == 0 || descriptors.cols == 0) { + return correspondence_result; + } + + int knn = 1; + std::vector> matches; + bf_matcher_.knnMatch(descriptors, feature_descriptors_[box_idx], matches, + knn); + + // Hamming distance threshold for best match distance. This max distance + // filtering rejects some of false matches which has not been rejected by + // cross match validation. And the value is determined emprically. + for (const auto &match_pair : matches) { + if (match_pair.size() < knn) continue; + const cv::DMatch &best_match = match_pair[0]; + if (best_match.distance > options_.max_match_distance()) continue; + + int match_idx = feature_to_frame_[box_idx][best_match.trainIdx]; + + correspondence_result[match_idx].points_frame.push_back(cv::Point2f( + features[best_match.queryIdx].x(), features[best_match.queryIdx].y())); + correspondence_result[match_idx].points_index.push_back( + cv::Point2f(feature_keypoints_[box_idx][best_match.trainIdx].x(), + feature_keypoints_[box_idx][best_match.trainIdx].y())); + } + + return correspondence_result; +} + +} // namespace mediapipe diff --git a/mediapipe/util/tracking/box_detector.h b/mediapipe/util/tracking/box_detector.h new file mode 100644 index 000000000..27c098212 --- /dev/null +++ b/mediapipe/util/tracking/box_detector.h @@ -0,0 +1,161 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_UTIL_TRACKING_BOX_DETECTOR_H_ +#define MEDIAPIPE_UTIL_TRACKING_BOX_DETECTOR_H_ + +#include "absl/container/flat_hash_map.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_features2d_inc.h" +#include "mediapipe/util/tracking/box_detector.pb.h" +#include "mediapipe/util/tracking/box_tracker.pb.h" +#include "mediapipe/util/tracking/flow_packager.pb.h" +#include "mediapipe/util/tracking/tracking.h" + +namespace mediapipe { + +// Feature correspondences between target index and a specific frame. +// The size of `points_frame` and `points_index` should be identical and the +// corresponding elements are a pair of feature correspondence. +struct FeatureCorrespondence { + // Matched feature locations from an image frame. + std::vector points_frame; + // Matched feature locations from the index structure. The location is where + // it get detected from a previous frame. + std::vector points_index; +}; + +// General interface for multiple box detector implementations +class BoxDetectorInterface { + public: + // Creates box detector based on index type defined in `options`. + static std::unique_ptr Create( + const BoxDetectorOptions &options); + + // Locate quad from feature correspondences using perspective model. + // Feature locations need to be normalized with 1.0 / max(width, height). + // `box_proto` contains quad corners position and aspect ratio. + // `frame_aspect` is the aspect ratio for the camera image frame. + // Note that to perform pnp tracking, both box aspect ratio and frame aspect + // ratio need to be positive. Otherwise fallback to homography tracking. + TimedBoxProtoList FindQuadFromFeatureCorrespondence( + const FeatureCorrespondence &matches, const TimedBoxProto &box_proto, + float frame_aspect = -1.0f); + + virtual ~BoxDetectorInterface() = default; + + // Detects pre-set boxes from input frame and adds features from new boxes + // into detector's index structure. Features and descriptors should be + // pre-computed and passed within `tracking_data`. `tracked_boxes` contains + // box tracking results from box_tracker. + // If all the boxes in the index are currently being tracked (box.id() found + // in `tracked_boxes`), the detection will be skipped and `detected_boxes` + // will remain empty. + // If the box's ID has never been recorded in the index before, The ID and all + // the features within the box will be merged into the index. + // `timestamp_msec` should correspond to `tracking_data`. + void DetectAndAddBox(const TrackingData &tracking_data, + const TimedBoxProtoList &tracked_boxes, + int64 timestamp_msec, TimedBoxProtoList *detected_boxes); + + // Detects pre-set boxes from input frame and adds features from new boxes + // into detector's index structure. Features and descriptors are extracted + // from `image` in real time. + // Other parameters work the same way as the previous function. + // `timestamp_msec` should correspond to `image`. + void DetectAndAddBox(const cv::Mat &image, + const TimedBoxProtoList &tracked_boxes, + int64 timestamp_msec, TimedBoxProtoList *detected_boxes); + + // Stops detection of box with `box_id`. + void CancelBoxDetection(int box_id); + + // Get the current detector's search index. + BoxDetectorIndex ObtainBoxDetectorIndex() const; + + // Add detector's search index with pre-defined index. + void AddBoxDetectorIndex(const BoxDetectorIndex &index); + + // Internal call for public DetectAndAddBox functions. `features` and + // `descriptors` can be either extracted from live frames or tracked from + // previous frames. `scale_x` and `scale_y` provides actual image aspect ratio + // so that boxes from `tracked_boxes` can be denormalized and boxes in + // `detected_boxes` normalized. `timestamp_msec` should correspond to the + // timestamp of `features` and `descriptors`. + void DetectAndAddBoxFromFeatures(const std::vector &features, + const cv::Mat &descriptors, + const TimedBoxProtoList &tracked_boxes, + int64 timestamp_msec, float scale_x, + float scale_y, + TimedBoxProtoList *detected_boxes); + + protected: + explicit BoxDetectorInterface(const BoxDetectorOptions &options); + + // `transform_features_for_pnp` controls wheather we transform features + // coordinates into a rectangular target space for pnp detection mode. + void AddBoxFeaturesToIndex(const std::vector &features, + const cv::Mat &descriptors, + const TimedBoxProto &box, + bool transform_features_for_pnp = false); + + // Check if add / detect action will be called based on input `tracked_boxes`. + bool CheckDetectAndAddBox(const TimedBoxProtoList &tracked_boxes); + + // Returns feature indices that are within the given box. If the box size + // isn't big enough to cover sufficient features to reacquire the box, this + // function will try to iteratively enlarge the box size by roughly 5 + // percent of the shorter edge of the image to include more features, but + // maximimum twice. Note that detected_boxes will still be reported with + // original size. External users are then freed from specificially finetuning + // a box size for reacquisition. They should choose suitable box size for + // tracking based on their use cases. + std::vector GetFeatureIndexWithinBox( + const std::vector &features, const TimedBoxProto &box); + + // Specifies which box to detect with `box_idx`. This enalbles separately + // managing the detection behavior for each box in the index. Tracked boxes + // will be skipped and lost and out-of-view boxes will be detected. + TimedBoxProtoList DetectBox(const std::vector &features, + const cv::Mat &descriptors, int box_idx); + + // Only matches those features from the specific box with `box_idx`. + virtual std::vector MatchFeatureDescriptors( + const std::vector &features, const cv::Mat &descriptors, + int box_idx) = 0; + + // Specifies which box the correspondences come from with `box_id`, so that we + // can figure out the transformation accordingly. + TimedBoxProtoList FindBoxesFromFeatureCorrespondence( + const std::vector &matches, int box_idx); + + int cnt_detect_called_ = 0; + float image_scale_; + float image_aspect_; + absl::flat_hash_map box_id_to_idx_; + std::vector box_idx_to_id_; + std::vector> frame_box_; + std::vector> feature_to_frame_; + std::vector> feature_keypoints_; + std::vector feature_descriptors_; + std::vector has_been_out_of_fov_; + mutable absl::Mutex access_to_index_; + cv::Ptr orb_extractor_; + BoxDetectorOptions options_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_UTIL_TRACKING_BOX_DETECTOR_H_ diff --git a/mediapipe/util/tracking/box_detector.proto b/mediapipe/util/tracking/box_detector.proto new file mode 100644 index 000000000..7b522f98a --- /dev/null +++ b/mediapipe/util/tracking/box_detector.proto @@ -0,0 +1,97 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/util/tracking/box_tracker.proto"; +import "mediapipe/util/tracking/region_flow.proto"; + +option java_package = "com.google.mediapipe.tracking"; +option java_outer_classname = "BoxDetectorProto"; + +message BoxDetectorOptions { + // Available types of detector's index and search structure. + enum IndexType { + INDEX_UNSPECIFIED = 0; + // BFMatcher from OpenCV + OPENCV_BF = 1; + } + + optional IndexType index_type = 1 [default = OPENCV_BF]; + + // Decide whether we force detector run every N frame. + // 0 means detection will never be called. + // 1 means detect every frame. 2 means detect every other frame. etc.. + // Currently only applied to image query mode. + optional int32 detect_every_n_frame = 2 [default = 0]; + + // Enable box detection when tracked boxes is out of FOV. Detection will be + // ceased after the detector successfully re-acquire the box. + optional bool detect_out_of_fov = 4 [default = false]; + + // Options only for detection from image queries. + message ImageQuerySettings { + // Resize the input image's longer edge to this size. Skip resizing if the + // input size is already smaller than this size. + optional int32 pyramid_bottom_size = 1 [default = 640]; + + // Scale factor between adjacent pyramid levels. + optional float pyramid_scale_factor = 2 [default = 1.2]; + + // Maximum number of pyramid levels. + optional int32 max_pyramid_levels = 3 [default = 4]; + + // Max number of features the detector uses. + optional int32 max_features = 4 [default = 500]; + } + + // Options for detection function with image query. + optional ImageQuerySettings image_query_settings = 3; + + // Dimensions (number of elements) for feature descriptor. + optional int32 descriptor_dims = 5 [default = 40]; + + // Minimum number of correspondence to go through RANSAC. + optional int32 min_num_correspondence = 6 [default = 5]; + + // Reprojection threshold for RANSAC to find inliers. + optional float ransac_reprojection_threshold = 7 [default = 0.02]; + + // Max distance to match 2 NIMBY features. + optional float max_match_distance = 8 [default = 0.8]; + + // Max persepective change factor. + optional float max_perspective_factor = 9 [default = 0.1]; +} + +// Proto to hold BoxDetector's internal search index. +message BoxDetectorIndex { + // Message to hold keypoints and descriptors for each box. + message BoxEntry { + // Message to hold keypoints and descriptors for each appearance. One box + // could have multiple appearances to account for shape and perspective + // change, etc.. + message FrameEntry { + optional TimedBoxProto box = 1; + repeated float keypoints = 2; + repeated BinaryFeatureDescriptor descriptors = 3; + } + + repeated FrameEntry frame_entry = 1; + } + + repeated BoxEntry box_entry = 1; +} diff --git a/mediapipe/util/tracking/flow_packager.cc b/mediapipe/util/tracking/flow_packager.cc index 3a570d85b..c236435f7 100644 --- a/mediapipe/util/tracking/flow_packager.cc +++ b/mediapipe/util/tracking/flow_packager.cc @@ -577,18 +577,16 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, : flow_compressed_8.size(); int32 row_idx_size = row_idx.size(); - absl::StrAppend( - data, - absl::StrCat(EncodeToString(frame_flags), EncodeToString(domain_width), - EncodeToString(domain_height), EncodeToString(frame_aspect), - background_model_string, EncodeToString(scale), - EncodeToString(num_vectors), - EncodeVectorToString(col_start_delta), - EncodeToString(row_idx_size), EncodeVectorToString(row_idx), - EncodeToString(vector_size), - (options_.high_fidelity_16bit_encode() - ? EncodeVectorToString(flow_compressed_16) - : EncodeVectorToString(flow_compressed_8)))); + absl::StrAppend(data, EncodeToString(frame_flags), + EncodeToString(domain_width), EncodeToString(domain_height), + EncodeToString(frame_aspect), background_model_string, + EncodeToString(scale), EncodeToString(num_vectors), + EncodeVectorToString(col_start_delta), + EncodeToString(row_idx_size), EncodeVectorToString(row_idx), + EncodeToString(vector_size), + (options_.high_fidelity_16bit_encode() + ? EncodeVectorToString(flow_compressed_16) + : EncodeVectorToString(flow_compressed_8))); VLOG(1) << "Binary data size: " << data->size() << " for " << num_vectors << " (" << vector_size << ")"; } @@ -862,9 +860,8 @@ void FlowPackager::FinalizeTrackingContainerFormat( std::string* binary_metadata = meta->mutable_data(); absl::StrAppend(binary_metadata, EncodeToString(meta_data.num_frames())); for (auto& track_offset : *meta_data.mutable_track_offsets()) { - absl::StrAppend(binary_metadata, - absl::StrCat(EncodeToString(track_offset.msec()), - EncodeToString(track_offset.stream_offset()))); + absl::StrAppend(binary_metadata, EncodeToString(track_offset.msec()), + EncodeToString(track_offset.stream_offset())); } meta->set_size(binary_metadata->size()); diff --git a/mediapipe/util/tracking/image_util.cc b/mediapipe/util/tracking/image_util.cc index cafdef079..90ac1ac7b 100644 --- a/mediapipe/util/tracking/image_util.cc +++ b/mediapipe/util/tracking/image_util.cc @@ -106,7 +106,7 @@ void RenderSaliency(const SalientPointFrame& salient_points, for (int k = 0; k < 4; ++k) { cv::line(*image, corners[k], corners[(k + 1) % 4], line_color, - line_thickness, CV_AA); + line_thickness, cv::LINE_AA); } } } diff --git a/mediapipe/util/tracking/motion_analysis.cc b/mediapipe/util/tracking/motion_analysis.cc index ff4d3d531..0b7678889 100644 --- a/mediapipe/util/tracking/motion_analysis.cc +++ b/mediapipe/util/tracking/motion_analysis.cc @@ -661,13 +661,13 @@ void MotionAnalysis::RenderResults(const RegionFlowFeatureList& feature_list, cv::putText(*rendered_results, hud_text, cv::Point(0.02 * frame_width_, 0.975 * frame_height_), cv::FONT_HERSHEY_SIMPLEX, text_scale, cv::Scalar::all(255), - text_scale * 3, CV_AA); + text_scale * 3, cv::LINE_AA); cv::putText(*rendered_results, absl::StrFormat("%6d", motion.timestamp_usec() / 1000), cv::Point(0.9 * frame_width_, 0.05 * frame_height_), cv::FONT_HERSHEY_SIMPLEX, text_scale, cv::Scalar::all(255), - text_scale * 3, CV_AA); + text_scale * 3, cv::LINE_AA); } #else LOG(FATAL) << "Code stripped out because of NO_RENDERING"; @@ -783,7 +783,7 @@ void MotionAnalysis::VisualizeBlurAnalysisRegions(cv::Mat* input_view) { CHECK(input_view != nullptr); cv::Mat intensity; - cv::cvtColor(*input_view, intensity, CV_RGB2GRAY); + cv::cvtColor(*input_view, intensity, cv::COLOR_RGB2GRAY); cv::Mat corner_values; cv::cornerMinEigenVal(intensity, corner_values, 3); diff --git a/mediapipe/util/tracking/region_flow_computation.cc b/mediapipe/util/tracking/region_flow_computation.cc index 331246220..58b2750b9 100644 --- a/mediapipe/util/tracking/region_flow_computation.cc +++ b/mediapipe/util/tracking/region_flow_computation.cc @@ -962,7 +962,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, LOG(ERROR) << "Expecting 3 channel input for RGB."; return false; } - cv::cvtColor(*source_ptr, dest_frame, CV_RGB2GRAY); + cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_RGB2GRAY); break; case RegionFlowComputationOptions::FORMAT_BGR: @@ -970,7 +970,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, LOG(ERROR) << "Expecting 3 channel input for BGR."; return false; } - cv::cvtColor(*source_ptr, dest_frame, CV_BGR2GRAY); + cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_BGR2GRAY); break; case RegionFlowComputationOptions::FORMAT_RGBA: @@ -978,7 +978,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, LOG(ERROR) << "Expecting 4 channel input for RGBA."; return false; } - cv::cvtColor(*source_ptr, dest_frame, CV_RGBA2GRAY); + cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_RGBA2GRAY); break; case RegionFlowComputationOptions::FORMAT_BGRA: @@ -986,7 +986,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, LOG(ERROR) << "Expecting 4 channel input for BGRA."; return false; } - cv::cvtColor(*source_ptr, dest_frame, CV_BGRA2GRAY); + cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_BGRA2GRAY); break; case RegionFlowComputationOptions::FORMAT_GRAYSCALE: diff --git a/mediapipe/util/tracking/region_flow_computation_test.cc b/mediapipe/util/tracking/region_flow_computation_test.cc index 256cfefbd..66319990c 100644 --- a/mediapipe/util/tracking/region_flow_computation_test.cc +++ b/mediapipe/util/tracking/region_flow_computation_test.cc @@ -152,19 +152,19 @@ void RegionFlowComputationTest::MakeMovie( break; case RegionFlowComputationOptions::FORMAT_BGR: - convert(CV_8UC3, CV_RGB2BGR); + convert(CV_8UC3, cv::COLOR_RGB2BGR); break; case RegionFlowComputationOptions::FORMAT_GRAYSCALE: - convert(CV_8UC1, CV_RGB2GRAY); + convert(CV_8UC1, cv::COLOR_RGB2GRAY); break; case RegionFlowComputationOptions::FORMAT_RGBA: - convert(CV_8UC4, CV_RGB2RGBA); + convert(CV_8UC4, cv::COLOR_RGB2RGBA); break; case RegionFlowComputationOptions::FORMAT_BGRA: - convert(CV_8UC4, CV_RGB2BGRA); + convert(CV_8UC4, cv::COLOR_RGB2BGRA); break; } for (int f = 0; f < num_frames; ++f) { diff --git a/mediapipe/util/tracking/region_flow_visualization.cc b/mediapipe/util/tracking/region_flow_visualization.cc index 8d36b92a5..dc067da7c 100644 --- a/mediapipe/util/tracking/region_flow_visualization.cc +++ b/mediapipe/util/tracking/region_flow_visualization.cc @@ -102,13 +102,13 @@ void VisualizeRegionFlowFeaturesImpl(const RegionFlowFeatureList& feature_list, #else cv::Scalar color_scaled(color * alpha + outlier * (1.0f - alpha)); #endif - cv::line(*output, p1, p2, color_scaled, line_size, CV_AA); + cv::line(*output, p1, p2, color_scaled, line_size, cv::LINE_AA); cv::circle(*output, p1, 2.0 * line_size, color_scaled, line_size); if (feature.has_label()) { cv::putText(*output, absl::StrCat(" ", feature.label()), p1, cv::FONT_HERSHEY_SIMPLEX, text_scale, color_scaled, - 3.0 * text_scale, CV_AA); + 3.0 * text_scale, cv::LINE_AA); } } } @@ -165,7 +165,7 @@ void VisualizeLongFeatureStreamImpl(const LongFeatureStream& stream, if (k < pts.size() - 1) { // Draw line connecting points. cv::Point p2(pts[k + 1].x() * scale_x, pts[k + 1].y() * scale_y); - cv::line(*output, p1, p2, color_scaled, 1.0, CV_AA); + cv::line(*output, p1, p2, color_scaled, 1.0, cv::LINE_AA); } if (k + 1 == pts.size()) { // Last iteration. cv::circle(*output, p1, 2.0, color_scaled, 1.0); @@ -174,7 +174,7 @@ void VisualizeLongFeatureStreamImpl(const LongFeatureStream& stream, if (latest_feature.has_label()) { cv::putText(*output, absl::StrCat(" ", latest_feature.label()), p1, cv::FONT_HERSHEY_SIMPLEX, text_scale, color_scaled, - 3 * text_scale, CV_AA); + 3 * text_scale, cv::LINE_AA); } } } diff --git a/mediapipe/util/tracking/tone_estimation.cc b/mediapipe/util/tracking/tone_estimation.cc index fd6e8edfe..02da9cb82 100644 --- a/mediapipe/util/tracking/tone_estimation.cc +++ b/mediapipe/util/tracking/tone_estimation.cc @@ -149,7 +149,7 @@ void ToneEstimation::IntensityPercentiles(const cv::Mat& frame, bool log_domain, ToneChange* tone_change) const { cv::Mat intensity(frame.rows, frame.cols, CV_8UC1); - cv::cvtColor(frame, intensity, CV_RGB2GRAY); + cv::cvtColor(frame, intensity, cv::COLOR_RGB2GRAY); std::vector histogram(256, 0.0f); diff --git a/mediapipe/util/tracking/tracked_detection.cc b/mediapipe/util/tracking/tracked_detection.cc index 3127a9e44..130a87640 100644 --- a/mediapipe/util/tracking/tracked_detection.cc +++ b/mediapipe/util/tracking/tracked_detection.cc @@ -90,7 +90,9 @@ void TrackedDetection::AddLabel(const std::string& label, float score) { } } -bool TrackedDetection::IsSameAs(const TrackedDetection& other) const { +bool TrackedDetection::IsSameAs(const TrackedDetection& other, + float max_area_ratio, + float min_overlap_ratio) const { const auto box0 = bounding_box_; const auto box1 = other.bounding_box_; const double box0_area = BoxArea(box0); @@ -104,14 +106,12 @@ bool TrackedDetection::IsSameAs(const TrackedDetection& other) const { // long water bottle) vertically and then change the camera to horizontal // quickly, then it will get another detection which will have a diamond shape // that is much larger than the previous rectangle one. - const double kMaxAreaRatio = 3.0; - if (box0_area / box1_area > kMaxAreaRatio || - box1_area / box0_area > kMaxAreaRatio) + if (box0_area / box1_area > max_area_ratio || + box1_area / box0_area > max_area_ratio) return false; - const double kMaxOverlapRatio = 0.5; - if (overlap_area / box0_area > kMaxOverlapRatio || - overlap_area / box1_area > kMaxOverlapRatio) { + if (overlap_area / box0_area > min_overlap_ratio || + overlap_area / box1_area > min_overlap_ratio) { return true; } diff --git a/mediapipe/util/tracking/tracked_detection.h b/mediapipe/util/tracking/tracked_detection.h index d4d0feee4..52032b631 100644 --- a/mediapipe/util/tracking/tracked_detection.h +++ b/mediapipe/util/tracking/tracked_detection.h @@ -54,7 +54,8 @@ class TrackedDetection { // Checks if two detections are in fact the same object. // TODO: This is just the first step. It will be expand to a family // of functionalities that do reconciliation. - bool IsSameAs(const TrackedDetection& other) const; + bool IsSameAs(const TrackedDetection& other, float max_area_ratio = 3.0f, + float min_overlap_ratio = 0.5f) const; // Merges labels and score from another TrackedDetection. void MergeLabelScore(const TrackedDetection& other); diff --git a/mediapipe/util/tracking/tracked_detection_manager.cc b/mediapipe/util/tracking/tracked_detection_manager.cc index 4a822ea6b..a5896bbf6 100644 --- a/mediapipe/util/tracking/tracked_detection_manager.cc +++ b/mediapipe/util/tracking/tracked_detection_manager.cc @@ -53,7 +53,9 @@ std::vector TrackedDetectionManager::AddDetection( // same if the timestamp are the same. for (auto& existing_detection_ptr : detections_) { const auto& existing_detection = *existing_detection_ptr.second; - if (detection->IsSameAs(existing_detection)) { + if (detection->IsSameAs(existing_detection, + config_.is_same_detection_max_area_ratio(), + config_.is_same_detection_min_overlap_ratio())) { // Merge previous labels to the new detection. Because new detections // usually have better bounding box than the one from tracking. // TODO: This might cause unstable change of the bounding box. @@ -148,7 +150,9 @@ std::vector TrackedDetectionManager::RemoveDuplicatedDetections(int id) { // locations of detections at different timestamp is not correct. if (detection.last_updated_timestamp() == other.last_updated_timestamp()) { - if (detection.IsSameAs(other)) { + if (detection.IsSameAs(other, + config_.is_same_detection_max_area_ratio(), + config_.is_same_detection_min_overlap_ratio())) { const TrackedDetection* detection_to_remove = nullptr; if (latest_detection->initial_timestamp() >= other.initial_timestamp()) { diff --git a/mediapipe/util/tracking/tracked_detection_manager.h b/mediapipe/util/tracking/tracked_detection_manager.h index d42d7c17c..8dfc2ac2a 100644 --- a/mediapipe/util/tracking/tracked_detection_manager.h +++ b/mediapipe/util/tracking/tracked_detection_manager.h @@ -18,6 +18,7 @@ #include "absl/container/node_hash_map.h" #include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/util/tracking/tracked_detection.h" +#include "mediapipe/util/tracking/tracked_detection_manager_config.pb.h" namespace mediapipe { @@ -68,6 +69,10 @@ class TrackedDetectionManager { return detections_; } + void SetConfig(const mediapipe::TrackedDetectionManagerConfig& config) { + config_ = config; + } + private: // Finds all detections that are duplicated with the one of |id| and remove // all detections except the one that is added most recently. Returns the IDs @@ -75,6 +80,8 @@ class TrackedDetectionManager { std::vector RemoveDuplicatedDetections(int id); absl::node_hash_map> detections_; + + mediapipe::TrackedDetectionManagerConfig config_; }; } // namespace mediapipe diff --git a/mediapipe/util/tracking/tracked_detection_manager_config.proto b/mediapipe/util/tracking/tracked_detection_manager_config.proto new file mode 100644 index 000000000..9162c954f --- /dev/null +++ b/mediapipe/util/tracking/tracked_detection_manager_config.proto @@ -0,0 +1,28 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +message TrackedDetectionManagerConfig { + // When we compare two detection boxes, if the ratio of the area is + // larger than is_same_detection_max_area_ratio, we consider them being + // different detections. + optional float is_same_detection_max_area_ratio = 1 [default = 3.0]; + // When we compare two detection boxes, if the overlap ratio is larger + // than is_same_detection_min_overlap_ratio, we consider them being + // same detection. + optional float is_same_detection_min_overlap_ratio = 2 [default = 0.5]; +} diff --git a/mediapipe/util/tracking/tracking_visualization_utilities.cc b/mediapipe/util/tracking/tracking_visualization_utilities.cc index c3ec91563..0b5860879 100644 --- a/mediapipe/util/tracking/tracking_visualization_utilities.cc +++ b/mediapipe/util/tracking/tracking_visualization_utilities.cc @@ -165,7 +165,7 @@ void RenderInternalState(const MotionBoxInternalState& internal, 255 * alpha + (1.0f - alpha) * 0, 0 * alpha + (1.0f - alpha) * 0); - cv::line(*frame, p1, p2, color_scaled, 1, CV_AA); + cv::line(*frame, p1, p2, color_scaled, 1, cv::LINE_AA); cv::circle(*frame, p1, 2.0, color_scaled, 1); } #else @@ -196,7 +196,7 @@ void RenderTrackingData(const TrackingData& data, cv::Mat* mat, // iOS cannot display a width 1 antialiased line, so we provide // an option for 8-connected drawing instead. cv::line(*mat, p1, p2, cv::Scalar(0, 255, 0, 255), 1, - antialiasing ? CV_AA : 8); + antialiasing ? cv::LINE_AA : 8); } #else LOG(FATAL) << "Code stripped out because of NO_RENDERING"; diff --git a/setup_opencv.sh b/setup_opencv.sh index 0a235529a..04b219c97 100644 --- a/setup_opencv.sh +++ b/setup_opencv.sh @@ -16,56 +16,72 @@ # # Script to build OpenCV from source code and modify the MediaPipe opencv config. # Note that this script only has been tested on Debian 9 and Ubuntu 16.04. -# usage: +# +# To have a full installation: # $ cd -# $ chmod +x ./setup_opencv.sh -# $ ./setup_opencv.sh +# $ sh ./setup_opencv.sh +# +# To only modify the mediapipe config for opencv: +# $ cd +# $ sh ./setup_opencv.sh config_only + set -e +if [ "$1" ] && [ "$1" != "config_only" ] + then + echo "Unknown input argument. Do you mean \"config_only\"?" + exit 0 +fi -opencv_build_file="$( cd "$(dirname "$0")" ; pwd -P )"/third_party/opencv_linux.BUILD -echo $opencv_build_file +if [ -z "$1" ] + then + echo "Installing OpenCV from source" + sudo apt update && sudo apt install build-essential git + sudo apt install cmake ffmpeg libavformat-dev libdc1394-22-dev libgtk2.0-dev \ + libjpeg-dev libpng-dev libswscale-dev libtbb2 libtbb-dev \ + libtiff-dev + rm -rf /tmp/build_opencv + mkdir /tmp/build_opencv + cd /tmp/build_opencv + git clone https://github.com/opencv/opencv_contrib.git + git clone https://github.com/opencv/opencv.git + mkdir opencv/release + cd opencv_contrib + git checkout 3.4 + cd ../opencv + git checkout 3.4 + cd release + cmake .. -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF -DBUILD_opencv_ts=OFF \ + -DOPENCV_EXTRA_MODULES_PATH=/tmp/build_opencv/opencv_contrib/modules \ + -DBUILD_opencv_aruco=OFF -DBUILD_opencv_bgsegm=OFF -DBUILD_opencv_bioinspired=OFF \ + -DBUILD_opencv_ccalib=OFF -DBUILD_opencv_datasets=OFF -DBUILD_opencv_dnn=OFF \ + -DBUILD_opencv_dnn_objdetect=OFF -DBUILD_opencv_dpm=OFF -DBUILD_opencv_face=OFF \ + -DBUILD_opencv_fuzzy=OFF -DBUILD_opencv_hfs=OFF -DBUILD_opencv_img_hash=OFF \ + -DBUILD_opencv_js=OFF -DBUILD_opencv_line_descriptor=OFF -DBUILD_opencv_phase_unwrapping=OFF \ + -DBUILD_opencv_plot=OFF -DBUILD_opencv_quality=OFF -DBUILD_opencv_reg=OFF \ + -DBUILD_opencv_rgbd=OFF -DBUILD_opencv_saliency=OFF -DBUILD_opencv_shape=OFF \ + -DBUILD_opencv_structured_light=OFF -DBUILD_opencv_surface_matching=OFF \ + -DBUILD_opencv_world=OFF -DBUILD_opencv_xobjdetect=OFF -DBUILD_opencv_xphoto=OFF + make -j 16 + sudo make install + rm -rf /tmp/build_opencv + echo "OpenCV has been built. You can find the header files and libraries in /usr/local/include/opencv2/ and /usr/local/lib" -echo "Installing OpenCV from source" -sudo apt update && sudo apt install build-essential git -sudo apt install cmake ffmpeg libavformat-dev libdc1394-22-dev libgtk2.0-dev \ - libjpeg-dev libpng-dev libswscale-dev libtbb2 libtbb-dev \ - libtiff-dev -rm -rf /tmp/build_opencv -mkdir /tmp/build_opencv -cd /tmp/build_opencv -git clone https://github.com/opencv/opencv_contrib.git -git clone https://github.com/opencv/opencv.git -mkdir opencv/release -cd opencv_contrib -git checkout 3.4 -cd ../opencv -git checkout 3.4 -cd release -cmake .. -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF -DBUILD_opencv_ts=OFF \ - -DOPENCV_EXTRA_MODULES_PATH=/tmp/build_opencv/opencv_contrib/modules \ - -DBUILD_opencv_aruco=OFF -DBUILD_opencv_bgsegm=OFF -DBUILD_opencv_bioinspired=OFF \ - -DBUILD_opencv_ccalib=OFF -DBUILD_opencv_datasets=OFF -DBUILD_opencv_dnn=OFF \ - -DBUILD_opencv_dnn_objdetect=OFF -DBUILD_opencv_dpm=OFF -DBUILD_opencv_face=OFF \ - -DBUILD_opencv_fuzzy=OFF -DBUILD_opencv_hfs=OFF -DBUILD_opencv_img_hash=OFF \ - -DBUILD_opencv_js=OFF -DBUILD_opencv_line_descriptor=OFF -DBUILD_opencv_phase_unwrapping=OFF \ - -DBUILD_opencv_plot=OFF -DBUILD_opencv_quality=OFF -DBUILD_opencv_reg=OFF \ - -DBUILD_opencv_rgbd=OFF -DBUILD_opencv_saliency=OFF -DBUILD_opencv_shape=OFF \ - -DBUILD_opencv_structured_light=OFF -DBUILD_opencv_surface_matching=OFF \ - -DBUILD_opencv_world=OFF -DBUILD_opencv_xobjdetect=OFF -DBUILD_opencv_xphoto=OFF -make -j 16 -sudo make install -rm -rf /tmp/build_opencv -echo "OpenCV has been built. You can find the header files and libraries in /usr/local/include/opencv4/opencv2 and /usr/local/lib" + # https://github.com/cggos/dip_cvqt/issues/1#issuecomment-284103343 + sudo touch /etc/ld.so.conf.d/mp_opencv.conf + sudo bash -c "echo /usr/local/lib >> /etc/ld.so.conf.d/mp_opencv.conf" + sudo ldconfig -v +fi # Modify the build file. echo "Modifying MediaPipe opencv config" -sed -i "s/lib\/x86_64-linux-gnu/local\/lib/g" $opencv_build_file -sed -i "20i \ \"local/lib/libopencv_optflow.so*\"," $opencv_build_file -sed -i "/includes =/d" $opencv_build_file -sed -i "/hdrs =/d" $opencv_build_file -line_to_insert=$(grep -n 'linkstatic =' $opencv_build_file | awk -F ":" '{print $1}')'i' -sed -i "$line_to_insert \ includes = [\"local\/include\/opencv4\/\"]," $opencv_build_file -sed -i "$line_to_insert \ hdrs = glob([\"local\/include\/opencv4\/\*\*\/\*\.h\*\"])," $opencv_build_file +opencv_build_file="$( cd "$(dirname "$0")" ; pwd -P )"/third_party/opencv_linux.BUILD +workspace_file="$( cd "$(dirname "$0")" ; pwd -P )"/WORKSPACE + +sed -i "s/lib\/x86_64-linux-gnu/lib/g" $opencv_build_file +linux_opencv_config=$(grep -n 'linux_opencv' $workspace_file | awk -F ":" '{print $1}') +path_line=$((linux_opencv_config + 2)) +sed -i "$path_line d" $workspace_file +sed -i "$path_line i\ path = \"/usr/local\"," $workspace_file echo "Done" diff --git a/third_party/build_bazel_rules_apple_bypass_test_runner_check.diff b/third_party/build_bazel_rules_apple_bypass_test_runner_check.diff new file mode 100644 index 000000000..1ba394982 --- /dev/null +++ b/third_party/build_bazel_rules_apple_bypass_test_runner_check.diff @@ -0,0 +1,13 @@ +diff --git a/apple/internal/testing/apple_test_assembler.bzl b/apple/internal/testing/apple_test_assembler.bzl +index f613313..63241f6 100644 +--- a/apple/internal/testing/apple_test_assembler.bzl ++++ b/apple/internal/testing/apple_test_assembler.bzl +@@ -63,8 +63,6 @@ def _assemble(name, bundle_rule, test_rule, runner = None, runners = None, **kwa + """ + if runner != None and runners != None: + fail("Can't specify both runner and runners.") +- elif not runner and not runners: +- fail("Must specify one of runner or runners.") + + test_bundle_name = name + ".__internal__.__test_bundle" + diff --git a/third_party/opencv_linux.BUILD b/third_party/opencv_linux.BUILD index 4d2eb254e..893eb8871 100644 --- a/third_party/opencv_linux.BUILD +++ b/third_party/opencv_linux.BUILD @@ -8,7 +8,8 @@ exports_files(["LICENSE"]) # The following build rule assumes that OpenCV is installed by # 'apt-get install libopencv-core-dev libopencv-highgui-dev \' # ' libopencv-calib3d-dev libopencv-features2d-dev \' -# ' libopencv-imgproc-dev libopencv-video-dev' on Debian/Ubuntu. +# ' libopencv-imgproc-dev libopencv-video-dev' +# on Debian buster/Ubuntu 18.04. # If you install OpenCV separately, please modify the build rule accordingly. cc_library( name = "opencv", diff --git a/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff b/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff deleted file mode 100644 index 6b28b0548..000000000 --- a/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff +++ /dev/null @@ -1,65 +0,0 @@ -diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc -index 6bb2cafbbc..60250d6f92 100644 ---- a/tensorflow/c/tf_tensor.cc -+++ b/tensorflow/c/tf_tensor.cc -@@ -114,7 +114,7 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims, - if (elem_size > 0 && len < (elem_size * ret.NumElements())) { - return nullptr; - } -- return new TF_Tensor{std::make_unique(ret)}; -+ return new TF_Tensor{absl::make_unique(ret)}; - } - - TF_Tensor* TF_TensorMaybeMove(TF_Tensor* t) { -@@ -317,7 +317,7 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { - if (!tensor.CopyFrom(src, src.shape())) { - return nullptr; - } -- return new TF_Tensor{std::make_unique(tensor)}; -+ return new TF_Tensor{absl::make_unique(tensor)}; - } - // DT_STRING tensors require a copying since TF_Tensor.buffer expects a flatly - // encoded sequence of strings. -diff --git a/tensorflow/core/lib/monitoring/percentile_sampler.cc b/tensorflow/core/lib/monitoring/percentile_sampler.cc -index 988e50ded5..a8a412dc25 100644 ---- a/tensorflow/core/lib/monitoring/percentile_sampler.cc -+++ b/tensorflow/core/lib/monitoring/percentile_sampler.cc -@@ -29,7 +29,8 @@ namespace monitoring { - void PercentileSamplerCell::Add(double sample) { - uint64 nstime = EnvTime::NowNanos(); - mutex_lock l(mu_); -- samples_[next_position_] = {nstime, sample}; -+ samples_[next_position_].nstime = nstime; -+ samples_[next_position_].value = sample; - ++next_position_; - if (TF_PREDICT_FALSE(next_position_ >= samples_.size())) { - next_position_ = 0; -@@ -72,7 +73,9 @@ Percentiles PercentileSamplerCell::value() const { - size_t index = std::min( - static_cast(percentile * pct_samples.num_samples / 100.0), - pct_samples.num_samples - 1); -- PercentilePoint pct = {percentile, samples[index].value}; -+ PercentilePoint pct; -+ pct.percentile = percentile; -+ pct.value = samples[index].value; - pct_samples.points.push_back(pct); - } - } -diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl -index 8947038da8..75a1259b88 100755 ---- a/tensorflow/workspace.bzl -+++ b/tensorflow/workspace.bzl -@@ -927,9 +927,10 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): - # https://github.com/bazelbuild/rules_apple/releases - tf_http_archive( - name = "build_bazel_rules_apple", -- sha256 = "a045a436b642c70fb0c10ca84ff0fd2dcbd59cc89100d597a61e8374afafb366", -+ sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", -+ strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", - urls = [ -- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_apple/releases/download/0.18.0/rules_apple.0.18.0.tar.gz", -- "https://github.com/bazelbuild/rules_apple/releases/download/0.18.0/rules_apple.0.18.0.tar.gz", -+ "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", -+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", - ], - ) diff --git a/third_party/org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff b/third_party/org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff deleted file mode 100644 index 4c15c9aec..000000000 --- a/third_party/org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff +++ /dev/null @@ -1,112 +0,0 @@ -diff --git a/third_party/cpuinfo/BUILD.bazel b/third_party/cpuinfo/BUILD.bazel -index 8d89521612..6ea60acdda 100644 ---- a/third_party/cpuinfo/BUILD.bazel -+++ b/third_party/cpuinfo/BUILD.bazel -@@ -116,6 +111,8 @@ cc_library( - ":watchos_x86": COMMON_SRCS + X86_SRCS + MACH_SRCS + MACH_X86_SRCS, - ":watchos_armv7k": COMMON_SRCS + MACH_SRCS + MACH_ARM_SRCS, - ":watchos_arm64_32": COMMON_SRCS + MACH_SRCS + MACH_ARM_SRCS, -+ ":tvos_x86_64": COMMON_SRCS + X86_SRCS + MACH_SRCS + MACH_X86_SRCS, -+ ":tvos_arm64": COMMON_SRCS + MACH_SRCS + MACH_ARM_SRCS, - ":emscripten_wasm": COMMON_SRCS + EMSCRIPTEN_SRCS, - }), - copts = C99OPTS + [ -@@ -212,7 +209,7 @@ config_setting( - config_setting( - name = "ios_armv7", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "ios", - "cpu": "ios_armv7", - }, - ) -@@ -220,7 +217,7 @@ config_setting( - config_setting( - name = "ios_arm64", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "ios", - "cpu": "ios_arm64", - }, - ) -@@ -228,7 +225,7 @@ config_setting( - config_setting( - name = "ios_arm64e", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "ios", - "cpu": "ios_arm64e", - }, - ) -@@ -236,7 +233,7 @@ config_setting( - config_setting( - name = "ios_x86", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "ios", - "cpu": "ios_i386", - }, - ) -@@ -244,7 +241,7 @@ config_setting( - config_setting( - name = "ios_x86_64", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "ios", - "cpu": "ios_x86_64", - }, - ) -@@ -252,7 +249,7 @@ config_setting( - config_setting( - name = "watchos_armv7k", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "watchos", - "cpu": "watchos_armv7k", - }, - ) -@@ -260,7 +257,7 @@ config_setting( - config_setting( - name = "watchos_arm64_32", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "watchos", - "cpu": "watchos_arm64_32", - }, - ) -@@ -268,7 +265,7 @@ config_setting( - config_setting( - name = "watchos_x86", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "watchos", - "cpu": "watchos_i386", - }, - ) -@@ -276,7 +273,7 @@ config_setting( - config_setting( - name = "watchos_x86_64", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "watchos", - "cpu": "watchos_x86_64", - }, - ) -@@ -284,7 +281,7 @@ config_setting( - config_setting( - name = "tvos_arm64", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "tvos", - "cpu": "tvos_arm64", - }, - ) -@@ -292,7 +289,7 @@ config_setting( - config_setting( - name = "tvos_x86_64", - values = { -- "crosstool_top": "//tools/osx/crosstool:crosstool", -+ "apple_platform_type": "tvos", - "cpu": "tvos_x86_64", - }, - ) diff --git a/third_party/org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff b/third_party/org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff deleted file mode 100644 index f42f31d43..000000000 --- a/third_party/org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff +++ /dev/null @@ -1,3083 +0,0 @@ -diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl -index 8947038da8..4e588d718e 100755 ---- a/tensorflow/workspace.bzl -+++ b/tensorflow/workspace.bzl -@@ -144,11 +144,11 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): - - tf_http_archive( - name = "XNNPACK", -- sha256 = "8f29d32a35d5e12aa5f02d0ef9018c80f2c985cd1837493fdfa670d84dfe2e2b", -- strip_prefix = "XNNPACK-1498d1d4d0430480dfe5c4538049b4f789d29134", -+ sha256 = "190e61e50af3497bb46b8d936bd2d2d551a9aeedb02ff66388918408a54e216a", -+ strip_prefix = "XNNPACK-b18783570f0643560be641b193367d3906955141", - urls = [ -- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/1498d1d4d0430480dfe5c4538049b4f789d29134.zip", -- "https://github.com/google/XNNPACK/archive/1498d1d4d0430480dfe5c4538049b4f789d29134.zip", -+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/b18783570f0643560be641b193367d3906955141.zip", -+ "https://github.com/google/XNNPACK/archive/b18783570f0643560be641b193367d3906955141.zip", - ], - ) - -diff --git a/third_party/cpuinfo/BUILD.bazel b/third_party/cpuinfo/BUILD.bazel -index cea88aafbd..afa0b9798a 100644 ---- a/third_party/cpuinfo/BUILD.bazel -+++ b/third_party/cpuinfo/BUILD.bazel -@@ -42,7 +42,6 @@ ARM_SRCS = [ - # Platform-specific sources and headers - LINUX_SRCS = [ - "src/linux/cpulist.c", -- "src/linux/current.c", - "src/linux/multiline.c", - "src/linux/processors.c", - "src/linux/smallfile.c", -diff --git a/third_party/cpuinfo/cpuinfo.patch b/third_party/cpuinfo/cpuinfo.patch -new file mode 100644 -index 0000000000..a9fa0dde0e ---- /dev/null -+++ b/third_party/cpuinfo/cpuinfo.patch -@@ -0,0 +1,3016 @@ -+diff --git a/CMakeLists.txt b/CMakeLists.txt -+index de319ef..fefb60b 100644 -+--- a/CMakeLists.txt -++++ b/CMakeLists.txt -+@@ -179,7 +179,6 @@ IF(CPUINFO_SUPPORTED_PLATFORM) -+ LIST(APPEND CPUINFO_SRCS -+ src/linux/smallfile.c -+ src/linux/multiline.c -+- src/linux/current.c -+ src/linux/cpulist.c -+ src/linux/processors.c) -+ ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") -+diff --git a/CMakeLists.txt.orig b/CMakeLists.txt.orig -+deleted file mode 100644 -+index a71aede..0000000 -+--- a/CMakeLists.txt.orig -++++ /dev/null -+@@ -1,819 +0,0 @@ -+-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12 FATAL_ERROR) -+- -+-INCLUDE(GNUInstallDirs) -+- -+-# ---[ Project and semantic versioning. -+-PROJECT(cpuinfo C CXX) -+- -+-# ---[ Options. -+-SET(CPUINFO_LIBRARY_TYPE "default" CACHE STRING "Type of cpuinfo library (shared, static, or default) to build") -+-SET_PROPERTY(CACHE CPUINFO_LIBRARY_TYPE PROPERTY STRINGS default static shared) -+-SET(CPUINFO_RUNTIME_TYPE "default" CACHE STRING "Type of runtime library (shared, static, or default) to use") -+-SET_PROPERTY(CACHE CPUINFO_RUNTIME_TYPE PROPERTY STRINGS default static shared) -+-SET(CPUINFO_LOG_LEVEL "default" CACHE STRING "Minimum logging level (info with lower severity will be ignored)") -+-SET_PROPERTY(CACHE CPUINFO_LOG_LEVEL PROPERTY STRINGS default debug info warning error fatal none) -+-OPTION(CPUINFO_BUILD_TOOLS "Build command-line tools" ON) -+-OPTION(CPUINFO_BUILD_UNIT_TESTS "Build cpuinfo unit tests" ON) -+-OPTION(CPUINFO_BUILD_MOCK_TESTS "Build cpuinfo mock tests" ON) -+-OPTION(CPUINFO_BUILD_BENCHMARKS "Build cpuinfo micro-benchmarks" ON) -+- -+-# ---[ CMake options -+-IF(CPUINFO_BUILD_UNIT_TESTS OR CPUINFO_BUILD_MOCK_TESTS) -+- ENABLE_TESTING() -+-ENDIF() -+- -+-MACRO(CPUINFO_TARGET_ENABLE_C99 target) -+- IF(${CMAKE_VERSION} VERSION_LESS "3.1") -+- IF(NOT MSVC) -+- TARGET_COMPILE_OPTIONS(${target} PRIVATE -std=c99) -+- ENDIF() -+- ELSE() -+- SET_TARGET_PROPERTIES(${target} PROPERTIES -+- C_STANDARD 99 -+- C_EXTENSIONS NO) -+- ENDIF() -+-ENDMACRO() -+- -+-MACRO(CPUINFO_TARGET_ENABLE_CXX11 target) -+- IF(${CMAKE_VERSION} VERSION_LESS "3.1") -+- IF(NOT MSVC) -+- TARGET_COMPILE_OPTIONS(${target} PRIVATE -std=c++11) -+- ENDIF() -+- ELSE() -+- SET_TARGET_PROPERTIES(${target} PROPERTIES -+- CXX_STANDARD 11 -+- CXX_EXTENSIONS NO) -+- ENDIF() -+-ENDMACRO() -+- -+-MACRO(CPUINFO_TARGET_RUNTIME_LIBRARY target) -+- IF(MSVC AND NOT CPUINFO_RUNTIME_TYPE STREQUAL "default") -+- IF(CPUINFO_RUNTIME_TYPE STREQUAL "shared") -+- TARGET_COMPILE_OPTIONS(${target} PRIVATE -+- "/MD$<$:d>") -+- ELSEIF(CPUINFO_RUNTIME_TYPE STREQUAL "static") -+- TARGET_COMPILE_OPTIONS(${target} PRIVATE -+- "/MT$<$:d>") -+- ENDIF() -+- ENDIF() -+-ENDMACRO() -+- -+-# ---[ Build flags -+-SET(CPUINFO_SUPPORTED_PLATFORM TRUE) -+-IF(NOT CMAKE_SYSTEM_PROCESSOR) -+- IF(NOT IOS) -+- MESSAGE(WARNING -+- "Target processor architecture is not specified. " -+- "cpuinfo will compile, but cpuinfo_initialize() will always fail.") -+- SET(CPUINFO_SUPPORTED_PLATFORM FALSE) -+- ENDIF() -+-ELSEIF(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86(_64)?|armv[5-8].*|aarch64)$") -+- MESSAGE(WARNING -+- "Target processor architecture \"${CMAKE_SYSTEM_PROCESSOR}\" is not supported in cpuinfo. " -+- "cpuinfo will compile, but cpuinfo_initialize() will always fail.") -+- SET(CPUINFO_SUPPORTED_PLATFORM FALSE) -+-ENDIF() -+- -+-IF(NOT CMAKE_SYSTEM_NAME) -+- MESSAGE(WARNING -+- "Target operating system is not specified. " -+- "cpuinfo will compile, but cpuinfo_initialize() will always fail.") -+- SET(CPUINFO_SUPPORTED_PLATFORM FALSE) -+-ELSEIF(NOT CMAKE_SYSTEM_NAME MATCHES "^(Windows|Darwin|Linux|Android)$") -+- IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14" AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") -+- MESSAGE(WARNING -+- "Target operating system \"${CMAKE_SYSTEM_NAME}\" is not supported in cpuinfo. " -+- "cpuinfo will compile, but cpuinfo_initialize() will always fail.") -+- SET(CPUINFO_SUPPORTED_PLATFORM FALSE) -+- ENDIF() -+-ENDIF() -+- -+-# ---[ Download deps -+-SET(CONFU_DEPENDENCIES_SOURCE_DIR ${CMAKE_SOURCE_DIR}/deps -+- CACHE PATH "Confu-style dependencies source directory") -+-SET(CONFU_DEPENDENCIES_BINARY_DIR ${CMAKE_BINARY_DIR}/deps -+- CACHE PATH "Confu-style dependencies binary directory") -+- -+-IF(CPUINFO_BUILD_MOCK_TESTS OR CPUINFO_BUILD_UNIT_TESTS) -+- IF(CPUINFO_SUPPORTED_PLATFORM AND NOT DEFINED GOOGLETEST_SOURCE_DIR) -+- MESSAGE(STATUS "Downloading Google Test to ${CONFU_DEPENDENCIES_SOURCE_DIR}/googletest (define GOOGLETEST_SOURCE_DIR to avoid it)") -+- CONFIGURE_FILE(cmake/DownloadGoogleTest.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest-download/CMakeLists.txt") -+- EXECUTE_PROCESS(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . -+- WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest-download") -+- EXECUTE_PROCESS(COMMAND "${CMAKE_COMMAND}" --build . -+- WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest-download") -+- SET(GOOGLETEST_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/googletest" CACHE STRING "Google Test source directory") -+- ENDIF() -+-ENDIF() -+- -+-IF(CPUINFO_BUILD_BENCHMARKS) -+- IF(CPUINFO_SUPPORTED_PLATFORM AND NOT DEFINED GOOGLEBENCHMARK_SOURCE_DIR) -+- MESSAGE(STATUS "Downloading Google Benchmark to ${CONFU_DEPENDENCIES_SOURCE_DIR}/googlebenchmark (define GOOGLEBENCHMARK_SOURCE_DIR to avoid it)") -+- CONFIGURE_FILE(cmake/DownloadGoogleBenchmark.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/googlebenchmark-download/CMakeLists.txt") -+- EXECUTE_PROCESS(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . -+- WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/googlebenchmark-download") -+- EXECUTE_PROCESS(COMMAND "${CMAKE_COMMAND}" --build . -+- WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/googlebenchmark-download") -+- SET(GOOGLEBENCHMARK_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/googlebenchmark" CACHE STRING "Google Benchmark source directory") -+- ENDIF() -+-ENDIF() -+- -+-# ---[ cpuinfo library -+-SET(CPUINFO_SRCS -+- src/init.c -+- src/api.c) -+- -+-IF(CPUINFO_SUPPORTED_PLATFORM) -+- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86(_64)?)$" OR IOS_ARCH MATCHES "^(i386|x86_64)$") -+- LIST(APPEND CPUINFO_SRCS -+- src/x86/init.c -+- src/x86/info.c -+- src/x86/vendor.c -+- src/x86/uarch.c -+- src/x86/name.c -+- src/x86/topology.c -+- src/x86/isa.c -+- src/x86/cache/init.c -+- src/x86/cache/descriptor.c -+- src/x86/cache/deterministic.c) -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- LIST(APPEND CPUINFO_SRCS -+- src/x86/linux/init.c -+- src/x86/linux/cpuinfo.c) -+- ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") -+- LIST(APPEND CPUINFO_SRCS src/x86/mach/init.c) -+- ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Windows") -+- LIST(APPEND CPUINFO_SRCS src/x86/windows/init.c) -+- ENDIF() -+- ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$" OR IOS_ARCH MATCHES "^(armv7.*|arm64.*)$") -+- LIST(APPEND CPUINFO_SRCS -+- src/arm/uarch.c -+- src/arm/cache.c) -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- LIST(APPEND CPUINFO_SRCS -+- src/arm/linux/init.c -+- src/arm/linux/cpuinfo.c -+- src/arm/linux/clusters.c -+- src/arm/linux/chipset.c -+- src/arm/linux/midr.c -+- src/arm/linux/hwcap.c) -+- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^armv[5-8]") -+- LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch32-isa.c) -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND ANDROID_ABI STREQUAL "armeabi") -+- SET_SOURCE_FILES_PROPERTIES(src/arm/linux/aarch32-isa.c PROPERTIES COMPILE_FLAGS -marm) -+- ENDIF() -+- ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") -+- LIST(APPEND CPUINFO_SRCS src/arm/linux/aarch64-isa.c) -+- ENDIF() -+- ELSEIF(IOS) -+- LIST(APPEND CPUINFO_SRCS src/arm/mach/init.c) -+- ENDIF() -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android") -+- LIST(APPEND CPUINFO_SRCS -+- src/arm/android/properties.c) -+- ENDIF() -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- LIST(APPEND CPUINFO_SRCS -+- src/linux/smallfile.c -+- src/linux/multiline.c -+- src/linux/current.c -+- src/linux/cpulist.c -+- src/linux/processors.c) -+- ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin" OR CMAKE_SYSTEM_NAME STREQUAL "iOS") -+- LIST(APPEND CPUINFO_SRCS src/mach/topology.c) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- SET(CMAKE_THREAD_PREFER_PTHREAD TRUE) -+- SET(THREADS_PREFER_PTHREAD_FLAG TRUE) -+- FIND_PACKAGE(Threads REQUIRED) -+- ENDIF() -+-ENDIF() -+- -+-IF(CPUINFO_LIBRARY_TYPE STREQUAL "default") -+- ADD_LIBRARY(cpuinfo ${CPUINFO_SRCS}) -+-ELSEIF(CPUINFO_LIBRARY_TYPE STREQUAL "shared") -+- ADD_LIBRARY(cpuinfo SHARED ${CPUINFO_SRCS}) -+-ELSEIF(CPUINFO_LIBRARY_TYPE STREQUAL "static") -+- ADD_LIBRARY(cpuinfo STATIC ${CPUINFO_SRCS}) -+-ELSE() -+- MESSAGE(FATAL_ERROR "Unsupported library type ${CPUINFO_LIBRARY_TYPE}") -+-ENDIF() -+-ADD_LIBRARY(cpuinfo_internals STATIC ${CPUINFO_SRCS}) -+-CPUINFO_TARGET_ENABLE_C99(cpuinfo) -+-CPUINFO_TARGET_ENABLE_C99(cpuinfo_internals) -+-CPUINFO_TARGET_RUNTIME_LIBRARY(cpuinfo) -+-SET_TARGET_PROPERTIES(cpuinfo PROPERTIES PUBLIC_HEADER include/cpuinfo.h) -+-TARGET_INCLUDE_DIRECTORIES(cpuinfo BEFORE PUBLIC include) -+-TARGET_INCLUDE_DIRECTORIES(cpuinfo BEFORE PRIVATE src) -+-TARGET_INCLUDE_DIRECTORIES(cpuinfo_internals BEFORE PUBLIC include src) -+-IF(CPUINFO_LOG_LEVEL STREQUAL "default") -+- # default logging level: error (subject to change) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=2) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "debug") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=5) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "info") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=4) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "warning") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=3) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "error") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=2) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "fatal") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=1) -+-ELSEIF(CPUINFO_LOG_LEVEL STREQUAL "none") -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE CPUINFO_LOG_LEVEL=0) -+-ELSE() -+- MESSAGE(FATAL_ERROR "Unsupported logging level ${CPUINFO_LOG_LEVEL}") -+-ENDIF() -+-TARGET_COMPILE_DEFINITIONS(cpuinfo_internals PRIVATE CPUINFO_LOG_LEVEL=0) -+- -+-IF(CPUINFO_SUPPORTED_PLATFORM) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo INTERFACE CPUINFO_SUPPORTED_PLATFORM=1) -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- TARGET_LINK_LIBRARIES(cpuinfo PUBLIC ${CMAKE_THREAD_LIBS_INIT}) -+- TARGET_LINK_LIBRARIES(cpuinfo_internals PUBLIC ${CMAKE_THREAD_LIBS_INIT}) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo PRIVATE _GNU_SOURCE=1) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo_internals PRIVATE _GNU_SOURCE=1) -+- ENDIF() -+-ELSE() -+- TARGET_COMPILE_DEFINITIONS(cpuinfo INTERFACE CPUINFO_SUPPORTED_PLATFORM=0) -+-ENDIF() -+- -+-# ---[ cpuinfo dependencies: clog -+-IF(NOT DEFINED CLOG_SOURCE_DIR) -+- SET(CLOG_SOURCE_DIR "${PROJECT_SOURCE_DIR}/deps/clog") -+-ENDIF() -+-IF(NOT TARGET clog) -+- SET(CLOG_BUILD_TESTS OFF CACHE BOOL "") -+- SET(CLOG_RUNTIME_TYPE "${CPUINFO_RUNTIME_TYPE}" CACHE STRING "") -+- ADD_SUBDIRECTORY( -+- "${CLOG_SOURCE_DIR}") -+- # We build static version of clog but a dynamic library may indirectly depend on it -+- SET_PROPERTY(TARGET clog PROPERTY POSITION_INDEPENDENT_CODE ON) -+-ENDIF() -+-TARGET_LINK_LIBRARIES(cpuinfo PRIVATE clog) -+-TARGET_LINK_LIBRARIES(cpuinfo_internals PRIVATE clog) -+- -+-INSTALL(TARGETS cpuinfo -+- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -+- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -+- PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -+- -+-# ---[ cpuinfo micro-benchmarks -+-IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_BENCHMARKS) -+- # ---[ Build google benchmark -+- IF(NOT TARGET benchmark) -+- SET(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "") -+- ADD_SUBDIRECTORY( -+- "${GOOGLEBENCHMARK_SOURCE_DIR}" -+- "${CONFU_DEPENDENCIES_BINARY_DIR}/googlebenchmark") -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME MATCHES "^(Linux|Android)$") -+- ADD_EXECUTABLE(get-current-bench bench/get-current.cc) -+- TARGET_LINK_LIBRARIES(get-current-bench cpuinfo benchmark) -+- ENDIF() -+- -+- ADD_EXECUTABLE(init-bench bench/init.cc) -+- TARGET_LINK_LIBRARIES(init-bench cpuinfo benchmark) -+-ENDIF() -+- -+-IF(CPUINFO_SUPPORTED_PLATFORM) -+- IF(CPUINFO_BUILD_MOCK_TESTS OR CPUINFO_BUILD_UNIT_TESTS) -+- # ---[ Build google test -+- IF(NOT TARGET gtest) -+- IF(MSVC AND NOT CPUINFO_RUNTIME_TYPE STREQUAL "static") -+- SET(gtest_force_shared_crt ON CACHE BOOL "" FORCE) -+- ENDIF() -+- ADD_SUBDIRECTORY( -+- "${GOOGLETEST_SOURCE_DIR}" -+- "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest") -+- ENDIF() -+- ENDIF() -+-ENDIF() -+- -+-# ---[ cpuinfo mock library and mock tests -+-IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_MOCK_TESTS) -+- SET(CPUINFO_MOCK_SRCS "${CPUINFO_SRCS}") -+- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86_64)$") -+- LIST(APPEND CPUINFO_MOCK_SRCS src/x86/mockcpuid.c) -+- ENDIF() -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- LIST(APPEND CPUINFO_MOCK_SRCS src/linux/mockfile.c) -+- ENDIF() -+- -+- ADD_LIBRARY(cpuinfo_mock STATIC ${CPUINFO_MOCK_SRCS}) -+- CPUINFO_TARGET_ENABLE_C99(cpuinfo_mock) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cpuinfo_mock) -+- SET_TARGET_PROPERTIES(cpuinfo_mock PROPERTIES PUBLIC_HEADER include/cpuinfo.h) -+- TARGET_INCLUDE_DIRECTORIES(cpuinfo_mock BEFORE PUBLIC include) -+- TARGET_INCLUDE_DIRECTORIES(cpuinfo_mock BEFORE PRIVATE src) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo_mock PUBLIC CPUINFO_MOCK=1) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo_mock PRIVATE CLOG_LOG_TO_STDIO=1) -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- TARGET_LINK_LIBRARIES(cpuinfo_mock PUBLIC ${CMAKE_THREAD_LIBS_INIT}) -+- TARGET_COMPILE_DEFINITIONS(cpuinfo_mock PRIVATE _GNU_SOURCE=1) -+- ENDIF() -+- TARGET_LINK_LIBRARIES(cpuinfo_mock PRIVATE clog) -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a)$") -+- ADD_EXECUTABLE(atm7029b-tablet-test test/mock/atm7029b-tablet.cc) -+- TARGET_INCLUDE_DIRECTORIES(atm7029b-tablet-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(atm7029b-tablet-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(atm7029b-tablet-test atm7029b-tablet-test) -+- -+- ADD_EXECUTABLE(blu-r1-hd-test test/mock/blu-r1-hd.cc) -+- TARGET_INCLUDE_DIRECTORIES(blu-r1-hd-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(blu-r1-hd-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(blu-r1-hd-test blu-r1-hd-test) -+- -+- ADD_EXECUTABLE(galaxy-a3-2016-eu-test test/mock/galaxy-a3-2016-eu.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-a3-2016-eu-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-a3-2016-eu-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-a3-2016-eu-test galaxy-a3-2016-eu-test) -+- -+- ADD_EXECUTABLE(galaxy-a8-2016-duos-test test/mock/galaxy-a8-2016-duos.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-a8-2016-duos-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-a8-2016-duos-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-a8-2016-duos-test galaxy-a8-2016-duos-test) -+- -+- ADD_EXECUTABLE(galaxy-grand-prime-value-edition-test test/mock/galaxy-grand-prime-value-edition.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-grand-prime-value-edition-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-grand-prime-value-edition-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-grand-prime-value-edition-test galaxy-grand-prime-value-edition-test) -+- -+- ADD_EXECUTABLE(galaxy-j1-2016-test test/mock/galaxy-j1-2016.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-j1-2016-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-j1-2016-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-j1-2016-test galaxy-j1-2016-test) -+- -+- ADD_EXECUTABLE(galaxy-j5-test test/mock/galaxy-j5.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-j5-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-j5-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-j5-test galaxy-j5-test) -+- -+- ADD_EXECUTABLE(galaxy-j7-prime-test test/mock/galaxy-j7-prime.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-j7-prime-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-j7-prime-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-j7-prime-test galaxy-j7-prime-test) -+- -+- ADD_EXECUTABLE(galaxy-j7-tmobile-test test/mock/galaxy-j7-tmobile.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-j7-tmobile-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-j7-tmobile-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-j7-tmobile-test galaxy-j7-tmobile-test) -+- -+- ADD_EXECUTABLE(galaxy-j7-uae-test test/mock/galaxy-j7-uae.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-j7-uae-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-j7-uae-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-j7-uae-test galaxy-j7-uae-test) -+- -+- ADD_EXECUTABLE(galaxy-s3-us-test test/mock/galaxy-s3-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s3-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s3-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s3-us-test galaxy-s3-us-test) -+- -+- ADD_EXECUTABLE(galaxy-s4-us-test test/mock/galaxy-s4-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s4-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s4-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s4-us-test galaxy-s4-us-test) -+- -+- ADD_EXECUTABLE(galaxy-s5-global-test test/mock/galaxy-s5-global.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s5-global-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s5-global-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s5-global-test galaxy-s5-global-test) -+- -+- ADD_EXECUTABLE(galaxy-s5-us-test test/mock/galaxy-s5-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s5-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s5-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s5-us-test galaxy-s5-us-test) -+- -+- ADD_EXECUTABLE(galaxy-tab-3-7.0-test test/mock/galaxy-tab-3-7.0.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-tab-3-7.0-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-tab-3-7.0-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-tab-3-7.0-test galaxy-tab-3-7.0-test) -+- -+- ADD_EXECUTABLE(galaxy-tab-3-lite-test test/mock/galaxy-tab-3-lite.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-tab-3-lite-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-tab-3-lite-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-tab-3-lite-test galaxy-tab-3-lite-test) -+- -+- ADD_EXECUTABLE(galaxy-win-duos-test test/mock/galaxy-win-duos.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-win-duos-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-win-duos-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-win-duos-test galaxy-win-duos-test) -+- -+- ADD_EXECUTABLE(huawei-ascend-p7-test test/mock/huawei-ascend-p7.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-ascend-p7-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-ascend-p7-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-ascend-p7-test huawei-ascend-p7-test) -+- -+- ADD_EXECUTABLE(huawei-honor-6-test test/mock/huawei-honor-6.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-honor-6-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-honor-6-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-honor-6-test huawei-honor-6-test) -+- -+- ADD_EXECUTABLE(lenovo-a6600-plus-test test/mock/lenovo-a6600-plus.cc) -+- TARGET_INCLUDE_DIRECTORIES(lenovo-a6600-plus-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(lenovo-a6600-plus-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(lenovo-a6600-plus-test lenovo-a6600-plus-test) -+- -+- ADD_EXECUTABLE(lenovo-vibe-x2-test test/mock/lenovo-vibe-x2.cc) -+- TARGET_INCLUDE_DIRECTORIES(lenovo-vibe-x2-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(lenovo-vibe-x2-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(lenovo-vibe-x2-test lenovo-vibe-x2-test) -+- -+- ADD_EXECUTABLE(lg-k10-eu-test test/mock/lg-k10-eu.cc) -+- TARGET_INCLUDE_DIRECTORIES(lg-k10-eu-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(lg-k10-eu-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(lg-k10-eu-test lg-k10-eu-test) -+- -+- ADD_EXECUTABLE(lg-optimus-g-pro-test test/mock/lg-optimus-g-pro.cc) -+- TARGET_INCLUDE_DIRECTORIES(lg-optimus-g-pro-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(lg-optimus-g-pro-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(lg-optimus-g-pro-test lg-optimus-g-pro-test) -+- -+- ADD_EXECUTABLE(moto-e-gen1-test test/mock/moto-e-gen1.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-e-gen1-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-e-gen1-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-e-gen1-test moto-e-gen1-test) -+- -+- ADD_EXECUTABLE(moto-g-gen1-test test/mock/moto-g-gen1.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-g-gen1-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-g-gen1-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-g-gen1-test moto-g-gen1-test) -+- -+- ADD_EXECUTABLE(moto-g-gen2-test test/mock/moto-g-gen2.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-g-gen2-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-g-gen2-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-g-gen2-test moto-g-gen2-test) -+- -+- ADD_EXECUTABLE(moto-g-gen3-test test/mock/moto-g-gen3.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-g-gen3-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-g-gen3-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-g-gen3-test moto-g-gen3-test) -+- -+- ADD_EXECUTABLE(moto-g-gen4-test test/mock/moto-g-gen4.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-g-gen4-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-g-gen4-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-g-gen4-test moto-g-gen4-test) -+- -+- ADD_EXECUTABLE(moto-g-gen5-test test/mock/moto-g-gen5.cc) -+- TARGET_INCLUDE_DIRECTORIES(moto-g-gen5-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(moto-g-gen5-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(moto-g-gen5-test moto-g-gen5-test) -+- -+- ADD_EXECUTABLE(nexus-s-test test/mock/nexus-s.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus-s-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus-s-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus-s-test nexus-s-test) -+- -+- ADD_EXECUTABLE(nexus4-test test/mock/nexus4.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus4-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus4-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus4-test nexus4-test) -+- -+- ADD_EXECUTABLE(nexus6-test test/mock/nexus6.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus6-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus6-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus6-test nexus6-test) -+- -+- ADD_EXECUTABLE(nexus10-test test/mock/nexus10.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus10-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus10-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus10-test nexus10-test) -+- -+- ADD_EXECUTABLE(padcod-10.1-test test/mock/padcod-10.1.cc) -+- TARGET_INCLUDE_DIRECTORIES(padcod-10.1-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(padcod-10.1-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(padcod-10.1-test padcod-10.1-test) -+- -+- ADD_EXECUTABLE(xiaomi-redmi-2a-test test/mock/xiaomi-redmi-2a.cc) -+- TARGET_INCLUDE_DIRECTORIES(xiaomi-redmi-2a-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xiaomi-redmi-2a-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xiaomi-redmi-2a-test xiaomi-redmi-2a-test) -+- -+- ADD_EXECUTABLE(xperia-sl-test test/mock/xperia-sl.cc) -+- TARGET_INCLUDE_DIRECTORIES(xperia-sl-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xperia-sl-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xperia-sl-test xperia-sl-test) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv5te|armv7-a|aarch64)$") -+- ADD_EXECUTABLE(alcatel-revvl-test test/mock/alcatel-revvl.cc) -+- TARGET_INCLUDE_DIRECTORIES(alcatel-revvl-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(alcatel-revvl-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(alcatel-revvl-test alcatel-revvl-test) -+- -+- ADD_EXECUTABLE(galaxy-a8-2018-test test/mock/galaxy-a8-2018.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-a8-2018-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-a8-2018-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-a8-2018-test galaxy-a8-2018-test) -+- -+- ADD_EXECUTABLE(galaxy-c9-pro-test test/mock/galaxy-c9-pro.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-c9-pro-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-c9-pro-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-c9-pro-test galaxy-c9-pro-test) -+- -+- ADD_EXECUTABLE(galaxy-s6-test test/mock/galaxy-s6.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s6-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s6-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s6-test galaxy-s6-test) -+- -+- ADD_EXECUTABLE(galaxy-s7-us-test test/mock/galaxy-s7-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s7-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s7-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s7-us-test galaxy-s7-us-test) -+- -+- ADD_EXECUTABLE(galaxy-s7-global-test test/mock/galaxy-s7-global.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s7-global-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s7-global-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s7-global-test galaxy-s7-global-test) -+- -+- ADD_EXECUTABLE(galaxy-s8-us-test test/mock/galaxy-s8-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s8-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s8-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s8-us-test galaxy-s8-us-test) -+- -+- ADD_EXECUTABLE(galaxy-s8-global-test test/mock/galaxy-s8-global.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s8-global-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s8-global-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s8-global-test galaxy-s8-global-test) -+- -+- ADD_EXECUTABLE(galaxy-s9-us-test test/mock/galaxy-s9-us.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s9-us-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s9-us-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s9-us-test galaxy-s9-us-test) -+- -+- ADD_EXECUTABLE(galaxy-s9-global-test test/mock/galaxy-s9-global.cc) -+- TARGET_INCLUDE_DIRECTORIES(galaxy-s9-global-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(galaxy-s9-global-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(galaxy-s9-global-test galaxy-s9-global-test) -+- -+- ADD_EXECUTABLE(huawei-mate-8-test test/mock/huawei-mate-8.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-mate-8-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-mate-8-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-mate-8-test huawei-mate-8-test) -+- -+- ADD_EXECUTABLE(huawei-mate-9-test test/mock/huawei-mate-9.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-mate-9-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-mate-9-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-mate-9-test huawei-mate-9-test) -+- -+- ADD_EXECUTABLE(huawei-mate-10-test test/mock/huawei-mate-10.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-mate-10-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-mate-10-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-mate-10-test huawei-mate-10-test) -+- -+- ADD_EXECUTABLE(huawei-mate-20-test test/mock/huawei-mate-20.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-mate-20-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-mate-20-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-mate-20-test huawei-mate-20-test) -+- -+- ADD_EXECUTABLE(huawei-p8-lite-test test/mock/huawei-p8-lite.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-p8-lite-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-p8-lite-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-p8-lite-test huawei-p8-lite-test) -+- -+- ADD_EXECUTABLE(huawei-p9-lite-test test/mock/huawei-p9-lite.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-p9-lite-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-p9-lite-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-p9-lite-test huawei-p9-lite-test) -+- -+- ADD_EXECUTABLE(huawei-p20-pro-test test/mock/huawei-p20-pro.cc) -+- TARGET_INCLUDE_DIRECTORIES(huawei-p20-pro-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(huawei-p20-pro-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(huawei-p20-pro-test huawei-p20-pro-test) -+- -+- ADD_EXECUTABLE(iconia-one-10-test test/mock/iconia-one-10.cc) -+- TARGET_INCLUDE_DIRECTORIES(iconia-one-10-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(iconia-one-10-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(iconia-one-10-test iconia-one-10-test) -+- -+- ADD_EXECUTABLE(meizu-pro-6-test test/mock/meizu-pro-6.cc) -+- TARGET_INCLUDE_DIRECTORIES(meizu-pro-6-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(meizu-pro-6-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(meizu-pro-6-test meizu-pro-6-test) -+- -+- ADD_EXECUTABLE(meizu-pro-6s-test test/mock/meizu-pro-6s.cc) -+- TARGET_INCLUDE_DIRECTORIES(meizu-pro-6s-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(meizu-pro-6s-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(meizu-pro-6s-test meizu-pro-6s-test) -+- -+- ADD_EXECUTABLE(meizu-pro-7-plus-test test/mock/meizu-pro-7-plus.cc) -+- TARGET_INCLUDE_DIRECTORIES(meizu-pro-7-plus-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(meizu-pro-7-plus-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(meizu-pro-7-plus-test meizu-pro-7-plus-test) -+- -+- ADD_EXECUTABLE(nexus5x-test test/mock/nexus5x.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus5x-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus5x-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus5x-test nexus5x-test) -+- -+- ADD_EXECUTABLE(nexus6p-test test/mock/nexus6p.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus6p-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus6p-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus6p-test nexus6p-test) -+- -+- ADD_EXECUTABLE(nexus9-test test/mock/nexus9.cc) -+- TARGET_INCLUDE_DIRECTORIES(nexus9-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(nexus9-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(nexus9-test nexus9-test) -+- -+- ADD_EXECUTABLE(oneplus-3t-test test/mock/oneplus-3t.cc) -+- TARGET_INCLUDE_DIRECTORIES(oneplus-3t-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oneplus-3t-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oneplus-3t-test oneplus-3t-test) -+- -+- ADD_EXECUTABLE(oneplus-5-test test/mock/oneplus-5.cc) -+- TARGET_INCLUDE_DIRECTORIES(oneplus-5-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oneplus-5-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oneplus-5-test oneplus-5-test) -+- -+- ADD_EXECUTABLE(oneplus-5t-test test/mock/oneplus-5t.cc) -+- TARGET_INCLUDE_DIRECTORIES(oneplus-5t-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oneplus-5t-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oneplus-5t-test oneplus-5t-test) -+- -+- ADD_EXECUTABLE(oppo-a37-test test/mock/oppo-a37.cc) -+- TARGET_INCLUDE_DIRECTORIES(oppo-a37-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oppo-a37-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oppo-a37-test oppo-a37-test) -+- -+- ADD_EXECUTABLE(oppo-r9-test test/mock/oppo-r9.cc) -+- TARGET_INCLUDE_DIRECTORIES(oppo-r9-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oppo-r9-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oppo-r9-test oppo-r9-test) -+- -+- ADD_EXECUTABLE(oppo-r15-test test/mock/oppo-r15.cc) -+- TARGET_INCLUDE_DIRECTORIES(oppo-r15-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(oppo-r15-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(oppo-r15-test oppo-r15-test) -+- -+- ADD_EXECUTABLE(pixel-test test/mock/pixel.cc) -+- TARGET_INCLUDE_DIRECTORIES(pixel-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(pixel-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(pixel-test pixel-test) -+- -+- ADD_EXECUTABLE(pixel-c-test test/mock/pixel-c.cc) -+- TARGET_INCLUDE_DIRECTORIES(pixel-c-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(pixel-c-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(pixel-c-test pixel-c-test) -+- -+- ADD_EXECUTABLE(pixel-xl-test test/mock/pixel-xl.cc) -+- TARGET_INCLUDE_DIRECTORIES(pixel-xl-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(pixel-xl-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(pixel-xl-test pixel-xl-test) -+- -+- ADD_EXECUTABLE(pixel-2-xl-test test/mock/pixel-2-xl.cc) -+- TARGET_INCLUDE_DIRECTORIES(pixel-2-xl-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(pixel-2-xl-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(pixel-2-xl-test pixel-2-xl-test) -+- -+- ADD_EXECUTABLE(xiaomi-mi-5c-test test/mock/xiaomi-mi-5c.cc) -+- TARGET_INCLUDE_DIRECTORIES(xiaomi-mi-5c-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xiaomi-mi-5c-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xiaomi-mi-5c-test xiaomi-mi-5c-test) -+- -+- ADD_EXECUTABLE(xiaomi-redmi-note-3-test test/mock/xiaomi-redmi-note-3.cc) -+- TARGET_INCLUDE_DIRECTORIES(xiaomi-redmi-note-3-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xiaomi-redmi-note-3-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xiaomi-redmi-note-3-test xiaomi-redmi-note-3-test) -+- -+- ADD_EXECUTABLE(xiaomi-redmi-note-4-test test/mock/xiaomi-redmi-note-4.cc) -+- TARGET_INCLUDE_DIRECTORIES(xiaomi-redmi-note-4-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xiaomi-redmi-note-4-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xiaomi-redmi-note-4-test xiaomi-redmi-note-4-test) -+- -+- ADD_EXECUTABLE(xperia-c4-dual-test test/mock/xperia-c4-dual.cc) -+- TARGET_INCLUDE_DIRECTORIES(xperia-c4-dual-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(xperia-c4-dual-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(xperia-c4-dual-test xperia-c4-dual-test) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(i686|x86_64)$") -+- ADD_EXECUTABLE(alldocube-iwork8-test test/mock/alldocube-iwork8.cc) -+- TARGET_INCLUDE_DIRECTORIES(alldocube-iwork8-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(alldocube-iwork8-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(alldocube-iwork8-test alldocube-iwork8-test) -+- -+- ADD_EXECUTABLE(leagoo-t5c-test test/mock/leagoo-t5c.cc) -+- TARGET_INCLUDE_DIRECTORIES(leagoo-t5c-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(leagoo-t5c-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(leagoo-t5c-test leagoo-t5c-test) -+- -+- ADD_EXECUTABLE(memo-pad-7-test test/mock/memo-pad-7.cc) -+- TARGET_INCLUDE_DIRECTORIES(memo-pad-7-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(memo-pad-7-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(memo-pad-7-test memo-pad-7-test) -+- -+- ADD_EXECUTABLE(zenfone-c-test test/mock/zenfone-c.cc) -+- TARGET_INCLUDE_DIRECTORIES(zenfone-c-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(zenfone-c-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(zenfone-c-test zenfone-c-test) -+- -+- ADD_EXECUTABLE(zenfone-2-test test/mock/zenfone-2.cc) -+- TARGET_INCLUDE_DIRECTORIES(zenfone-2-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(zenfone-2-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(zenfone-2-test zenfone-2-test) -+- -+- ADD_EXECUTABLE(zenfone-2e-test test/mock/zenfone-2e.cc) -+- TARGET_INCLUDE_DIRECTORIES(zenfone-2e-test BEFORE PRIVATE test/mock) -+- TARGET_LINK_LIBRARIES(zenfone-2e-test PRIVATE cpuinfo_mock gtest) -+- ADD_TEST(zenfone-2e-test zenfone-2e-test) -+- ENDIF() -+-ENDIF() -+- -+-# ---[ cpuinfo unit tests -+-IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_UNIT_TESTS) -+- ADD_EXECUTABLE(init-test test/init.cc) -+- CPUINFO_TARGET_ENABLE_CXX11(init-test) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(init-test) -+- TARGET_LINK_LIBRARIES(init-test PRIVATE cpuinfo gtest gtest_main) -+- ADD_TEST(init-test init-test) -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android") -+- ADD_EXECUTABLE(get-current-test test/get-current.cc) -+- CPUINFO_TARGET_ENABLE_CXX11(get-current-test) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(get-current-test) -+- TARGET_LINK_LIBRARIES(get-current-test PRIVATE cpuinfo gtest gtest_main) -+- ADD_TEST(get-current-test get-current-test) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86_64)$") -+- ADD_EXECUTABLE(brand-string-test test/name/brand-string.cc) -+- CPUINFO_TARGET_ENABLE_CXX11(brand-string-test) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(brand-string-test) -+- TARGET_LINK_LIBRARIES(brand-string-test PRIVATE cpuinfo_internals gtest gtest_main) -+- ADD_TEST(brand-string-test brand-string-test) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_NAME STREQUAL "Android" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+- ADD_LIBRARY(android_properties_interface STATIC test/name/android-properties-interface.c) -+- CPUINFO_TARGET_ENABLE_C99(android_properties_interface) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(android_properties_interface) -+- TARGET_LINK_LIBRARIES(android_properties_interface PRIVATE cpuinfo_internals) -+- -+- ADD_EXECUTABLE(chipset-test -+- test/name/proc-cpuinfo-hardware.cc -+- test/name/ro-product-board.cc -+- test/name/ro-board-platform.cc -+- test/name/ro-mediatek-platform.cc -+- test/name/ro-arch.cc -+- test/name/ro-chipname.cc -+- test/name/android-properties.cc) -+- CPUINFO_TARGET_ENABLE_CXX11(chipset-test) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(chipset-test) -+- TARGET_LINK_LIBRARIES(chipset-test PRIVATE android_properties_interface gtest gtest_main) -+- ADD_TEST(chipset-test chipset-test) -+- -+- ADD_EXECUTABLE(cache-test test/arm-cache.cc) -+- CPUINFO_TARGET_ENABLE_CXX11(cache-test) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cache-test) -+- TARGET_COMPILE_DEFINITIONS(cache-test PRIVATE __STDC_LIMIT_MACROS=1 __STDC_CONSTANT_MACROS=1) -+- TARGET_LINK_LIBRARIES(cache-test PRIVATE cpuinfo_internals gtest gtest_main) -+- ADD_TEST(cache-test, cache-test) -+- ENDIF() -+-ENDIF() -+- -+-# ---[ Helper and debug tools -+-IF(CPUINFO_SUPPORTED_PLATFORM AND CPUINFO_BUILD_TOOLS) -+- ADD_EXECUTABLE(isa-info tools/isa-info.c) -+- CPUINFO_TARGET_ENABLE_C99(isa-info) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(isa-info) -+- TARGET_LINK_LIBRARIES(isa-info PRIVATE cpuinfo) -+- INSTALL(TARGETS isa-info RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+- -+- ADD_EXECUTABLE(cpu-info tools/cpu-info.c) -+- CPUINFO_TARGET_ENABLE_C99(cpu-info) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cpu-info) -+- TARGET_LINK_LIBRARIES(cpu-info PRIVATE cpuinfo) -+- INSTALL(TARGETS cpu-info RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+- -+- ADD_EXECUTABLE(cache-info tools/cache-info.c) -+- CPUINFO_TARGET_ENABLE_C99(cache-info) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cache-info) -+- TARGET_LINK_LIBRARIES(cache-info PRIVATE cpuinfo) -+- INSTALL(TARGETS cache-info RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+- -+- IF(CMAKE_SYSTEM_NAME MATCHES "^(Android|Linux)$" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv[5-8].*|aarch64)$") -+- ADD_EXECUTABLE(auxv-dump tools/auxv-dump.c) -+- CPUINFO_TARGET_ENABLE_C99(auxv-dump) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(auxv-dump) -+- TARGET_LINK_LIBRARIES(auxv-dump PRIVATE ${CMAKE_DL_LIBS} cpuinfo) -+- -+- ADD_EXECUTABLE(cpuinfo-dump tools/cpuinfo-dump.c) -+- CPUINFO_TARGET_ENABLE_C99(cpuinfo-dump) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cpuinfo-dump) -+- ENDIF() -+- -+- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86_64)$") -+- ADD_EXECUTABLE(cpuid-dump tools/cpuid-dump.c) -+- CPUINFO_TARGET_ENABLE_C99(cpuid-dump) -+- CPUINFO_TARGET_RUNTIME_LIBRARY(cpuid-dump) -+- TARGET_INCLUDE_DIRECTORIES(cpuid-dump BEFORE PRIVATE src) -+- TARGET_INCLUDE_DIRECTORIES(cpuid-dump BEFORE PRIVATE include) -+- INSTALL(TARGETS cpuid-dump RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+- ENDIF() -+-ENDIF() -+diff --git a/README.md b/README.md -+index 7d383ff..ee5fb82 100644 -+--- a/README.md -++++ b/README.md -+@@ -152,21 +152,20 @@ pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpu_set); -+ - [x] Using `ro.chipname`, `ro.board.platform`, `ro.product.board`, `ro.mediatek.platform`, `ro.arch` properties (Android) -+ - [ ] Using kernel log (`dmesg`) on ARM Linux -+ - Vendor and microarchitecture detection -+- - [x] Intel-designed x86/x86-64 cores (up to Kaby Lake, Airmont, and Knights Mill) -+- - [x] AMD-designed x86/x86-64 cores (up to Puma/Jaguar and Zen) -++ - [x] Intel-designed x86/x86-64 cores (up to Sunny Cove, Goldmont Plus, and Knights Mill) -++ - [x] AMD-designed x86/x86-64 cores (up to Puma/Jaguar and Zen 2) -+ - [ ] VIA-designed x86/x86-64 cores -+ - [ ] Other x86 cores (DM&P, RDC, Transmeta, Cyrix, Rise) -+- - [x] ARM-designed ARM cores (up to Cortex-A55 and Cortex-A75) -+- - [x] Qualcomm-designed ARM cores (up to Kryo, Kryo-280, and Kryo-385) -+- - [x] Nvidia-designed ARM cores (Denver) -++ - [x] ARM-designed ARM cores (up to Cortex-A55, Cortex-A77, and Neoverse E1/N1) -++ - [x] Qualcomm-designed ARM cores (Scorpion, Krait, and Kryo) -++ - [x] Nvidia-designed ARM cores (Denver and Carmel) -+ - [x] Samsung-designed ARM cores (Exynos) -+ - [x] Intel-designed ARM cores (XScale up to 3rd-gen) -+- - [x] Apple-designed ARM cores (up to Hurricane) -++ - [x] Apple-designed ARM cores (up to Lightning and Thunder) -+ - [x] Cavium-designed ARM cores (ThunderX) -+ - [x] AppliedMicro-designed ARM cores (X-Gene) -+ - Instruction set detection -+ - [x] Using CPUID (x86/x86-64) -+- - [x] Using dynamic code generation validator (Native Client/x86-64) -+ - [x] Using `/proc/cpuinfo` on 32-bit ARM EABI (Linux) -+ - [x] Using microarchitecture heuristics on (32-bit ARM) -+ - [x] Using `FPSID` and `WCID` registers (32-bit ARM) -+diff --git a/bench/get-current.cc b/bench/get-current.cc -+index 91b35a0..b547df0 100644 -+--- a/bench/get-current.cc -++++ b/bench/get-current.cc -+@@ -21,4 +21,13 @@ static void cpuinfo_get_current_core(benchmark::State& state) { -+ } -+ BENCHMARK(cpuinfo_get_current_core)->Unit(benchmark::kNanosecond); -+ -++static void cpuinfo_get_current_uarch_index(benchmark::State& state) { -++ cpuinfo_initialize(); -++ while (state.KeepRunning()) { -++ const uint32_t uarch_index = cpuinfo_get_current_uarch_index(); -++ benchmark::DoNotOptimize(uarch_index); -++ } -++} -++BENCHMARK(cpuinfo_get_current_uarch_index)->Unit(benchmark::kNanosecond); -++ -+ BENCHMARK_MAIN(); -+diff --git a/cmake/DownloadGoogleTest.cmake b/cmake/DownloadGoogleTest.cmake -+index d69d19a..dc86c9c 100644 -+--- a/cmake/DownloadGoogleTest.cmake -++++ b/cmake/DownloadGoogleTest.cmake -+@@ -4,8 +4,8 @@ PROJECT(googletest-download NONE) -+ -+ INCLUDE(ExternalProject) -+ ExternalProject_Add(googletest -+- URL https://github.com/google/googletest/archive/release-1.8.0.zip -+- URL_HASH SHA256=f3ed3b58511efd272eb074a3a6d6fb79d7c2e6a0e374323d1e6bcbcc1ef141bf -++ URL https://github.com/google/googletest/archive/release-1.10.0.zip -++ URL_HASH SHA256=94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91 -+ SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/googletest" -+ BINARY_DIR "${CONFU_DEPENDENCIES_BINARY_DIR}/googletest" -+ CONFIGURE_COMMAND "" -+diff --git a/configure.py b/configure.py -+index a340c4c..0e58dba 100755 -+--- a/configure.py -++++ b/configure.py -+@@ -26,8 +26,8 @@ def main(args): -+ sources = ["init.c", "api.c"] -+ if build.target.is_x86 or build.target.is_x86_64: -+ sources += [ -+- "x86/init.c", "x86/info.c", "x86/vendor.c", "x86/uarch.c", "x86/name.c", -+- "x86/topology.c", -++ "x86/init.c", "x86/info.c", "x86/isa.c", "x86/vendor.c", -++ "x86/uarch.c", "x86/name.c", "x86/topology.c", -+ "x86/cache/init.c", "x86/cache/descriptor.c", "x86/cache/deterministic.c", -+ ] -+ if build.target.is_macos: -+@@ -37,7 +37,6 @@ def main(args): -+ "x86/linux/init.c", -+ "x86/linux/cpuinfo.c", -+ ] -+- sources.append("x86/isa.c" if not build.target.is_nacl else "x86/nacl/isa.c") -+ if build.target.is_arm or build.target.is_arm64: -+ sources += ["arm/uarch.c", "arm/cache.c"] -+ if build.target.is_linux or build.target.is_android: -+diff --git a/include/cpuinfo.h b/include/cpuinfo.h -+index 9938d2b..e4d2d0c 100644 -+--- a/include/cpuinfo.h -++++ b/include/cpuinfo.h -+@@ -34,10 +34,6 @@ -+ #define CPUINFO_ARCH_PPC64 1 -+ #endif -+ -+-#if defined(__pnacl__) -+- #define CPUINFO_ARCH_PNACL 1 -+-#endif -+- -+ #if defined(__asmjs__) -+ #define CPUINFO_ARCH_ASMJS 1 -+ #endif -+@@ -80,10 +76,6 @@ -+ #define CPUINFO_ARCH_PPC64 0 -+ #endif -+ -+-#ifndef CPUINFO_ARCH_PNACL -+- #define CPUINFO_ARCH_PNACL 0 -+-#endif -+- -+ #ifndef CPUINFO_ARCH_ASMJS -+ #define CPUINFO_ARCH_ASMJS 0 -+ #endif -+@@ -190,6 +182,12 @@ enum cpuinfo_vendor { -+ * Processors are designed by HiSilicon, a subsidiary of Huawei. -+ */ -+ cpuinfo_vendor_huawei = 15, -++ /** -++ * Hygon (Chengdu Haiguang Integrated Circuit Design Co., Ltd), Vendor of x86-64 processor microarchitectures. -++ * -++ * Processors are variants of AMD cores. -++ */ -++ cpuinfo_vendor_hygon = 16, -+ -+ /* Active vendors of embedded CPUs */ -+ -+@@ -401,6 +399,8 @@ enum cpuinfo_uarch { -+ cpuinfo_uarch_cortex_a35 = 0x00300335, -+ /** ARM Cortex-A53. */ -+ cpuinfo_uarch_cortex_a53 = 0x00300353, -++ /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities compared to revision 1+). */ -++ cpuinfo_uarch_cortex_a55r0 = 0x00300354, -+ /** ARM Cortex-A55. */ -+ cpuinfo_uarch_cortex_a55 = 0x00300355, -+ /** ARM Cortex-A57. */ -+@@ -478,6 +478,10 @@ enum cpuinfo_uarch { -+ cpuinfo_uarch_vortex = 0x00700107, -+ /** Apple A12 processor (little cores). */ -+ cpuinfo_uarch_tempest = 0x00700108, -++ /** Apple A13 processor (big cores). */ -++ cpuinfo_uarch_lightning = 0x00700109, -++ /** Apple A13 processor (little cores). */ -++ cpuinfo_uarch_thunder = 0x0070010A, -+ -+ /** Cavium ThunderX. */ -+ cpuinfo_uarch_thunderx = 0x00800100, -+@@ -494,6 +498,9 @@ enum cpuinfo_uarch { -+ -+ /** Applied Micro X-Gene. */ -+ cpuinfo_uarch_xgene = 0x00B00100, -++ -++ /* Hygon Dhyana (a modification of AMD Zen for Chinese market). */ -++ cpuinfo_uarch_dhyana = 0x01000100, -+ }; -+ -+ struct cpuinfo_processor { -+@@ -613,6 +620,22 @@ struct cpuinfo_package { -+ uint32_t cluster_count; -+ }; -+ -++struct cpuinfo_uarch_info { -++ /** Type of CPU microarchitecture */ -++ enum cpuinfo_uarch uarch; -++#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 -++ /** Value of CPUID leaf 1 EAX register for the microarchitecture */ -++ uint32_t cpuid; -++#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ /** Value of Main ID Register (MIDR) for the microarchitecture */ -++ uint32_t midr; -++#endif -++ /** Number of logical processors with the microarchitecture */ -++ uint32_t processor_count; -++ /** Number of cores with the microarchitecture */ -++ uint32_t core_count; -++}; -++ -+ #ifdef __cplusplus -+ extern "C" { -+ #endif -+@@ -1721,6 +1744,7 @@ const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processors(void); -+ const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_cores(void); -+ const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_clusters(void); -+ const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_packages(void); -++const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarchs(void); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void); -+@@ -1731,6 +1755,7 @@ const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processor(uint32_t index -+ const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_core(uint32_t index); -+ const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_cluster(uint32_t index); -+ const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_package(uint32_t index); -++const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarch(uint32_t index); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index); -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index); -+@@ -1741,6 +1766,7 @@ uint32_t CPUINFO_ABI cpuinfo_get_processors_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_cores_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_clusters_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_packages_count(void); -++uint32_t CPUINFO_ABI cpuinfo_get_uarchs_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void); -+ uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void); -+@@ -1752,9 +1778,31 @@ uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void); -+ */ -+ uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void); -+ -++/** -++ * Identify the logical processor that executes the current thread. -++ * -++ * There is no guarantee that the thread will stay on the same logical processor for any time. -++ * Callers should treat the result as only a hint, and be prepared to handle NULL return value. -++ */ -+ const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void); -++ -++/** -++ * Identify the core that executes the current thread. -++ * -++ * There is no guarantee that the thread will stay on the same core for any time. -++ * Callers should treat the result as only a hint, and be prepared to handle NULL return value. -++ */ -+ const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void); -+ -++/** -++ * Identify the microarchitecture index of the core that executes the current thread. -++ * If the system does not support such identification, the function return 0. -++ * -++ * There is no guarantee that the thread will stay on the same type of core for any time. -++ * Callers should treat the result as only a hint. -++ */ -++uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void); -++ -+ #ifdef __cplusplus -+ } /* extern "C" */ -+ #endif -+diff --git a/src/api.c b/src/api.c -+index b180d80..0cc5d4e 100644 -+--- a/src/api.c -++++ b/src/api.c -+@@ -1,9 +1,16 @@ -++#include -+ #include -+ -+ #include -+ #include -+ #include -+ -++#ifdef __linux__ -++ #include -++ -++ #include -++ #include -++#endif -+ -+ bool cpuinfo_is_initialized = false; -+ -+@@ -20,235 +27,347 @@ uint32_t cpuinfo_packages_count = 0; -+ uint32_t cpuinfo_cache_count[cpuinfo_cache_level_max] = { 0 }; -+ uint32_t cpuinfo_max_cache_size = 0; -+ -++#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ struct cpuinfo_uarch_info* cpuinfo_uarchs = NULL; -++ uint32_t cpuinfo_uarchs_count = 0; -++#else -++ struct cpuinfo_uarch_info cpuinfo_global_uarch = { cpuinfo_uarch_unknown }; -++#endif -++ -++#ifdef __linux__ -++ uint32_t cpuinfo_linux_cpu_max = 0; -++ const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map = NULL; -++ const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map = NULL; -++ #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ const uint32_t* cpuinfo_linux_cpu_to_uarch_index_map = NULL; -++ #endif -++#endif -++ -+ -+ const struct cpuinfo_processor* cpuinfo_get_processors(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processors"); -+ } -+ return cpuinfo_processors; -+ } -+ -+ const struct cpuinfo_core* cpuinfo_get_cores(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "core"); -+ } -+ return cpuinfo_cores; -+ } -+ -+ const struct cpuinfo_cluster* cpuinfo_get_clusters(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "clusters"); -+ } -+ return cpuinfo_clusters; -+ } -+ -+ const struct cpuinfo_package* cpuinfo_get_packages(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "packages"); -+ } -+ return cpuinfo_packages; -+ } -+ -+-const struct cpuinfo_processor* cpuinfo_get_processor(uint32_t index) { -++const struct cpuinfo_uarch_info* cpuinfo_get_uarchs() { -+ if (!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarchs"); -++ } -++ #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ return cpuinfo_uarchs; -++ #else -++ return &cpuinfo_global_uarch; -++ #endif -++} -++ -++const struct cpuinfo_processor* cpuinfo_get_processor(uint32_t index) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processor"); -+ } -+- if (index < cpuinfo_processors_count) { -+- return cpuinfo_processors + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_processors_count) { -+ return NULL; -+ } -++ return &cpuinfo_processors[index]; -+ } -+ -+ const struct cpuinfo_core* cpuinfo_get_core(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "core"); -+ } -+- if (index < cpuinfo_cores_count) { -+- return cpuinfo_cores + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cores_count) { -+ return NULL; -+ } -++ return &cpuinfo_cores[index]; -+ } -+ -+ const struct cpuinfo_cluster* cpuinfo_get_cluster(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "cluster"); -+ } -+- if (index < cpuinfo_clusters_count) { -+- return cpuinfo_clusters + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_clusters_count) { -+ return NULL; -+ } -++ return &cpuinfo_clusters[index]; -+ } -+ -+ const struct cpuinfo_package* cpuinfo_get_package(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "package"); -+ } -+- if (index < cpuinfo_packages_count) { -+- return cpuinfo_packages + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_packages_count) { -+ return NULL; -+ } -++ return &cpuinfo_packages[index]; -+ } -+ -+-uint32_t cpuinfo_get_processors_count(void) { -++const struct cpuinfo_uarch_info* cpuinfo_get_uarch(uint32_t index) { -+ if (!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarch"); -++ } -++ #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ if CPUINFO_UNLIKELY(index >= cpuinfo_uarchs_count) { -++ return NULL; -++ } -++ return &cpuinfo_uarchs[index]; -++ #else -++ if CPUINFO_UNLIKELY(index != 0) { -++ return NULL; -++ } -++ return &cpuinfo_global_uarch; -++ #endif -++} -++ -++uint32_t cpuinfo_get_processors_count(void) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "processors_count"); -+ } -+ return cpuinfo_processors_count; -+ } -+ -+ uint32_t cpuinfo_get_cores_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "cores_count"); -+ } -+ return cpuinfo_cores_count; -+ } -+ -+ uint32_t cpuinfo_get_clusters_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "clusters_count"); -+ } -+ return cpuinfo_clusters_count; -+ } -+ -+ uint32_t cpuinfo_get_packages_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "packages_count"); -+ } -+ return cpuinfo_packages_count; -+ } -+ -+-const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void) { -++uint32_t cpuinfo_get_uarchs_count(void) { -+ if (!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "uarchs_count"); -++ } -++ #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ return cpuinfo_uarchs_count; -++ #else -++ return 1; -++ #endif -++} -++ -++const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_caches"); -+ } -+ return cpuinfo_cache[cpuinfo_cache_level_1i]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_caches"); -+ } -+ return cpuinfo_cache[cpuinfo_cache_level_1d]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_caches"); -+ } -+ return cpuinfo_cache[cpuinfo_cache_level_2]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_caches(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_caches"); -+ } -+ return cpuinfo_cache[cpuinfo_cache_level_3]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_caches(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_caches"); -+ } -+ return cpuinfo_cache[cpuinfo_cache_level_4]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_cache"); -+ } -+- if (index < cpuinfo_cache_count[cpuinfo_cache_level_1i]) { -+- return cpuinfo_cache[cpuinfo_cache_level_1i] + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_1i]) { -+ return NULL; -+ } -++ return &cpuinfo_cache[cpuinfo_cache_level_1i][index]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_cache"); -+ } -+- if (index < cpuinfo_cache_count[cpuinfo_cache_level_1d]) { -+- return cpuinfo_cache[cpuinfo_cache_level_1d] + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_1d]) { -+ return NULL; -+ } -++ return &cpuinfo_cache[cpuinfo_cache_level_1d][index]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_cache"); -+ } -+- if (index < cpuinfo_cache_count[cpuinfo_cache_level_2]) { -+- return cpuinfo_cache[cpuinfo_cache_level_2] + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_2]) { -+ return NULL; -+ } -++ return &cpuinfo_cache[cpuinfo_cache_level_2][index]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_cache(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_cache"); -+ } -+- if (index < cpuinfo_cache_count[cpuinfo_cache_level_3]) { -+- return cpuinfo_cache[cpuinfo_cache_level_3] + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_3]) { -+ return NULL; -+ } -++ return &cpuinfo_cache[cpuinfo_cache_level_3][index]; -+ } -+ -+ const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_cache(uint32_t index) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_cache"); -+ } -+- if (index < cpuinfo_cache_count[cpuinfo_cache_level_4]) { -+- return cpuinfo_cache[cpuinfo_cache_level_4] + index; -+- } else { -++ if CPUINFO_UNLIKELY(index >= cpuinfo_cache_count[cpuinfo_cache_level_4]) { -+ return NULL; -+ } -++ return &cpuinfo_cache[cpuinfo_cache_level_4][index]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1i_caches_count"); -+ } -+ return cpuinfo_cache_count[cpuinfo_cache_level_1i]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l1d_caches_count"); -+ } -+ return cpuinfo_cache_count[cpuinfo_cache_level_1d]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l2_caches_count"); -+ } -+ return cpuinfo_cache_count[cpuinfo_cache_level_2]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_l3_caches_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l3_caches_count"); -+ } -+ return cpuinfo_cache_count[cpuinfo_cache_level_3]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "l4_caches_count"); -+ } -+ return cpuinfo_cache_count[cpuinfo_cache_level_4]; -+ } -+ -+ uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void) { -+- if (!cpuinfo_is_initialized) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -+ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "max_cache_size"); -+ } -+ return cpuinfo_max_cache_size; -+ } -++ -++const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_processor"); -++ } -++ #ifdef __linux__ -++ unsigned cpu; -++ if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { -++ return 0; -++ } -++ if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { -++ return 0; -++ } -++ return cpuinfo_linux_cpu_to_processor_map[cpu]; -++ #else -++ return NULL; -++ #endif -++} -++ -++const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_core"); -++ } -++ #ifdef __linux__ -++ unsigned cpu; -++ if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { -++ return 0; -++ } -++ if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { -++ return 0; -++ } -++ return cpuinfo_linux_cpu_to_core_map[cpu]; -++ #else -++ return NULL; -++ #endif -++} -++ -++uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void) { -++ if CPUINFO_UNLIKELY(!cpuinfo_is_initialized) { -++ cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_uarch_index"); -++ } -++ #if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ #ifdef __linux__ -++ if (cpuinfo_linux_cpu_to_uarch_index_map == NULL) { -++ /* Special case: avoid syscall on systems with only a single type of cores */ -++ return 0; -++ } -++ -++ /* General case */ -++ unsigned cpu; -++ if CPUINFO_UNLIKELY(syscall(__NR_getcpu, &cpu, NULL, NULL) != 0) { -++ return 0; -++ } -++ if CPUINFO_UNLIKELY((uint32_t) cpu >= cpuinfo_linux_cpu_max) { -++ return 0; -++ } -++ return cpuinfo_linux_cpu_to_uarch_index_map[cpu]; -++ #else -++ /* Fallback: pretend to be on the big core. */ -++ return 0; -++ #endif -++ #else -++ /* Only ARM/ARM64 processors may include cores of different types in the same package. */ -++ return 0; -++ #endif -++} -+diff --git a/src/arm/cache.c b/src/arm/cache.c -+index ccadeb4..c2bc7d2 100644 -+--- a/src/arm/cache.c -++++ b/src/arm/cache.c -+@@ -659,6 +659,7 @@ void cpuinfo_arm_decode_cache( -+ }; -+ } -+ break; -++ case cpuinfo_uarch_cortex_a55r0: -+ case cpuinfo_uarch_cortex_a55: -+ /* -+ * ARM Cortex-A55 Core Technical Reference Manual -+diff --git a/src/arm/linux/api.h b/src/arm/linux/api.h -+index 275d072..f99da66 100644 -+--- a/src/arm/linux/api.h -++++ b/src/arm/linux/api.h -+@@ -153,6 +153,7 @@ struct cpuinfo_arm_linux_processor { -+ uint32_t midr; -+ enum cpuinfo_vendor vendor; -+ enum cpuinfo_uarch uarch; -++ uint32_t uarch_index; -+ /** -+ * ID of the physical package which includes this logical processor. -+ * The value is parsed from /sys/devices/system/cpu/cpu/topology/physical_package_id -+@@ -346,3 +347,6 @@ CPUINFO_INTERNAL uint32_t cpuinfo_arm_linux_detect_cluster_midr( -+ uint32_t max_processors, -+ uint32_t usable_processors, -+ struct cpuinfo_arm_linux_processor processors[restrict static max_processors]); -++ -++extern CPUINFO_INTERNAL const uint32_t* cpuinfo_linux_cpu_to_uarch_index_map; -++extern CPUINFO_INTERNAL uint32_t cpuinfo_linux_cpu_to_uarch_index_map_entries; -+diff --git a/src/arm/linux/init.c b/src/arm/linux/init.c -+index f0c432c..6272abf 100644 -+--- a/src/arm/linux/init.c -++++ b/src/arm/linux/init.c -+@@ -106,12 +106,14 @@ void cpuinfo_arm_linux_init(void) { -+ struct cpuinfo_processor* processors = NULL; -+ struct cpuinfo_core* cores = NULL; -+ struct cpuinfo_cluster* clusters = NULL; -+- const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL; -+- const struct cpuinfo_core** linux_cpu_to_core_map = NULL; -++ struct cpuinfo_uarch_info* uarchs = NULL; -+ struct cpuinfo_cache* l1i = NULL; -+ struct cpuinfo_cache* l1d = NULL; -+ struct cpuinfo_cache* l2 = NULL; -+ struct cpuinfo_cache* l3 = NULL; -++ const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL; -++ const struct cpuinfo_core** linux_cpu_to_core_map = NULL; -++ uint32_t* linux_cpu_to_uarch_index_map = NULL; -+ -+ const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count(); -+ cpuinfo_log_debug("system maximum processors count: %"PRIu32, max_processors_count); -+@@ -400,6 +402,18 @@ void cpuinfo_arm_linux_init(void) { -+ } -+ } -+ -++ uint32_t uarchs_count = 0; -++ enum cpuinfo_uarch last_uarch; -++ for (uint32_t i = 0; i < arm_linux_processors_count; i++) { -++ if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { -++ if (uarchs_count == 0 || arm_linux_processors[i].uarch != last_uarch) { -++ last_uarch = arm_linux_processors[i].uarch; -++ uarchs_count += 1; -++ } -++ arm_linux_processors[i].uarch_index = uarchs_count - 1; -++ } -++ } -++ -+ /* -+ * Assumptions: -+ * - No SMP (i.e. each core supports only one hardware thread). -+@@ -432,6 +446,13 @@ void cpuinfo_arm_linux_init(void) { -+ goto cleanup; -+ } -+ -++ uarchs = calloc(uarchs_count, sizeof(struct cpuinfo_uarch_info)); -++ if (uarchs == NULL) { -++ cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" microarchitectures", -++ uarchs_count * sizeof(struct cpuinfo_uarch_info), uarchs_count); -++ goto cleanup; -++ } -++ -+ linux_cpu_to_processor_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_processor*)); -+ if (linux_cpu_to_processor_map == NULL) { -+ cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" logical processor mapping entries", -+@@ -446,6 +467,15 @@ void cpuinfo_arm_linux_init(void) { -+ goto cleanup; -+ } -+ -++ if (uarchs_count > 1) { -++ linux_cpu_to_uarch_index_map = calloc(arm_linux_processors_count, sizeof(uint32_t)); -++ if (linux_cpu_to_uarch_index_map == NULL) { -++ cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" uarch index mapping entries", -++ arm_linux_processors_count * sizeof(uint32_t), arm_linux_processors_count); -++ goto cleanup; -++ } -++ } -++ -+ l1i = calloc(valid_processors, sizeof(struct cpuinfo_cache)); -+ if (l1i == NULL) { -+ cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches", -+@@ -460,6 +490,22 @@ void cpuinfo_arm_linux_init(void) { -+ goto cleanup; -+ } -+ -++ uint32_t uarchs_index = 0; -++ for (uint32_t i = 0; i < arm_linux_processors_count; i++) { -++ if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) { -++ if (uarchs_index == 0 || arm_linux_processors[i].uarch != last_uarch) { -++ last_uarch = arm_linux_processors[i].uarch; -++ uarchs[uarchs_index] = (struct cpuinfo_uarch_info) { -++ .uarch = arm_linux_processors[i].uarch, -++ .midr = arm_linux_processors[i].midr, -++ }; -++ uarchs_index += 1; -++ } -++ uarchs[uarchs_index - 1].processor_count += 1; -++ uarchs[uarchs_index - 1].core_count += 1; -++ } -++ } -++ -+ uint32_t l2_count = 0, l3_count = 0, big_l3_size = 0, cluster_id = UINT32_MAX; -+ /* Indication whether L3 (if it exists) is shared between all cores */ -+ bool shared_l3 = true; -+@@ -499,6 +545,11 @@ void cpuinfo_arm_linux_init(void) { -+ cores[i].midr = arm_linux_processors[i].midr; -+ linux_cpu_to_core_map[arm_linux_processors[i].system_processor_id] = &cores[i]; -+ -++ if (linux_cpu_to_uarch_index_map != NULL) { -++ linux_cpu_to_uarch_index_map[arm_linux_processors[i].system_processor_id] = -++ arm_linux_processors[i].uarch_index; -++ } -++ -+ struct cpuinfo_cache temp_l2 = { 0 }, temp_l3 = { 0 }; -+ cpuinfo_arm_decode_cache( -+ arm_linux_processors[i].uarch, -+@@ -658,12 +709,11 @@ void cpuinfo_arm_linux_init(void) { -+ } -+ -+ /* Commit */ -+- cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; -+- cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; -+ cpuinfo_processors = processors; -+ cpuinfo_cores = cores; -+ cpuinfo_clusters = clusters; -+ cpuinfo_packages = &package; -++ cpuinfo_uarchs = uarchs; -+ cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; -+ cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; -+ cpuinfo_cache[cpuinfo_cache_level_2] = l2; -+@@ -673,33 +723,42 @@ void cpuinfo_arm_linux_init(void) { -+ cpuinfo_cores_count = valid_processors; -+ cpuinfo_clusters_count = cluster_count; -+ cpuinfo_packages_count = 1; -++ cpuinfo_uarchs_count = uarchs_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_1i] = valid_processors; -+ cpuinfo_cache_count[cpuinfo_cache_level_1d] = valid_processors; -+ cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -+- -+ cpuinfo_max_cache_size = cpuinfo_arm_compute_max_cache_size(&processors[0]); -+ -++ cpuinfo_linux_cpu_max = arm_linux_processors_count; -++ cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; -++ cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; -++ cpuinfo_linux_cpu_to_uarch_index_map = linux_cpu_to_uarch_index_map; -++ -+ __sync_synchronize(); -+ -+ cpuinfo_is_initialized = true; -+ -+- linux_cpu_to_processor_map = NULL; -+- linux_cpu_to_core_map = NULL; -+ processors = NULL; -+ cores = NULL; -+ clusters = NULL; -++ uarchs = NULL; -+ l1i = l1d = l2 = l3 = NULL; -++ linux_cpu_to_processor_map = NULL; -++ linux_cpu_to_core_map = NULL; -++ linux_cpu_to_uarch_index_map = NULL; -+ -+ cleanup: -+ free(arm_linux_processors); -+- free(linux_cpu_to_processor_map); -+- free(linux_cpu_to_core_map); -+ free(processors); -+ free(cores); -+ free(clusters); -++ free(uarchs); -+ free(l1i); -+ free(l1d); -+ free(l2); -+ free(l3); -++ free(linux_cpu_to_processor_map); -++ free(linux_cpu_to_core_map); -++ free(linux_cpu_to_uarch_index_map); -+ } -+diff --git a/src/arm/mach/init.c b/src/arm/mach/init.c -+index e64cc18..bd27259 100644 -+--- a/src/arm/mach/init.c -++++ b/src/arm/mach/init.c -+@@ -14,6 +14,16 @@ -+ #include -+ #include -+ -++/* Polyfill recent CPUFAMILY_ARM_* values for older SDKs */ -++#ifndef CPUFAMILY_ARM_MONSOON_MISTRAL -++ #define CPUFAMILY_ARM_MONSOON_MISTRAL 0xE81E7EF6 -++#endif -++#ifndef CPUFAMILY_ARM_VORTEX_TEMPEST -++ #define CPUFAMILY_ARM_VORTEX_TEMPEST 0x07D34B9F -++#endif -++#ifndef CPUFAMILY_ARM_LIGHTNING_THUNDER -++ #define CPUFAMILY_ARM_LIGHTNING_THUNDER 0x462504D2 -++#endif -+ -+ struct cpuinfo_arm_isa cpuinfo_isa = { -+ #if CPUINFO_ARCH_ARM -+@@ -82,37 +92,34 @@ static enum cpuinfo_uarch decode_uarch(uint32_t cpu_family, uint32_t cpu_subtype -+ return cpuinfo_uarch_twister; -+ case CPUFAMILY_ARM_HURRICANE: -+ return cpuinfo_uarch_hurricane; -+-#ifdef CPUFAMILY_ARM_MONSOON_MISTRAL -+ case CPUFAMILY_ARM_MONSOON_MISTRAL: -+-#else -+- case 0xe81e7ef6: -+- /* Hard-coded value for older SDKs which do not define CPUFAMILY_ARM_MONSOON_MISTRAL */ -+-#endif -+ /* 2x Monsoon + 4x Mistral cores */ -+ return core_index < 2 ? cpuinfo_uarch_monsoon : cpuinfo_uarch_mistral; -+-#ifdef CPUFAMILY_ARM_VORTEX_TEMPEST -+ case CPUFAMILY_ARM_VORTEX_TEMPEST: -+-#else -+- case 0x07d34b9f: -+- /* Hard-coded value for older SDKs which do not define CPUFAMILY_ARM_VORTEX_TEMPEST */ -+-#endif -+ /* Hexa-core: 2x Vortex + 4x Tempest; Octa-core: 4x Cortex + 4x Tempest */ -+ return core_index + 4 < core_count ? cpuinfo_uarch_vortex : cpuinfo_uarch_tempest; -++ case CPUFAMILY_ARM_LIGHTNING_THUNDER: -++ /* Hexa-core: 2x Lightning + 4x Thunder; Octa-core (presumed): 4x Lightning + 4x Thunder */ -++ return core_index + 4 < core_count ? cpuinfo_uarch_lightning : cpuinfo_uarch_thunder; -+ default: -+ /* Use hw.cpusubtype for detection */ -+ break; -+ } -+ -+- switch (cpu_subtype) { -+- case CPU_SUBTYPE_ARM_V7: -+- return cpuinfo_uarch_cortex_a8; -+- case CPU_SUBTYPE_ARM_V7F: -+- return cpuinfo_uarch_cortex_a9; -+- case CPU_SUBTYPE_ARM_V7K: -+- return cpuinfo_uarch_cortex_a7; -+- default: -+- return cpuinfo_uarch_unknown; -+- } -++ #if CPUINFO_ARCH_ARM -++ switch (cpu_subtype) { -++ case CPU_SUBTYPE_ARM_V7: -++ return cpuinfo_uarch_cortex_a8; -++ case CPU_SUBTYPE_ARM_V7F: -++ return cpuinfo_uarch_cortex_a9; -++ case CPU_SUBTYPE_ARM_V7K: -++ return cpuinfo_uarch_cortex_a7; -++ default: -++ return cpuinfo_uarch_unknown; -++ } -++ #else -++ return cpuinfo_uarch_unknown; -++ #endif -+ } -+ -+ static void decode_package_name(char* package_name) { -+@@ -244,6 +251,7 @@ void cpuinfo_arm_mach_init(void) { -+ struct cpuinfo_core* cores = NULL; -+ struct cpuinfo_cluster* clusters = NULL; -+ struct cpuinfo_package* packages = NULL; -++ struct cpuinfo_uarch_info* uarchs = NULL; -+ struct cpuinfo_cache* l1i = NULL; -+ struct cpuinfo_cache* l1d = NULL; -+ struct cpuinfo_cache* l2 = NULL; -+@@ -330,21 +338,12 @@ void cpuinfo_arm_mach_init(void) { -+ * Thus, we whitelist CPUs known to support these instructions. -+ */ -+ switch (cpu_family) { -+-#ifdef CPUFAMILY_ARM_MONSOON_MISTRAL -+ case CPUFAMILY_ARM_MONSOON_MISTRAL: -+-#else -+- case 0xe81e7ef6: -+- /* Hard-coded value for older SDKs which do not define CPUFAMILY_ARM_MONSOON_MISTRAL */ -+-#endif -+-#ifdef CPUFAMILY_ARM_VORTEX_TEMPEST -+ case CPUFAMILY_ARM_VORTEX_TEMPEST: -+-#else -+- case 0x07d34b9f: -+- /* Hard-coded value for older SDKs which do not define CPUFAMILY_ARM_VORTEX_TEMPEST */ -+-#endif -+-#if CPUINFO_ARCH_ARM64 -+- cpuinfo_isa.atomics = true; -+-#endif -++ case CPUFAMILY_ARM_LIGHTNING_THUNDER: -++ #if CPUINFO_ARCH_ARM64 -++ cpuinfo_isa.atomics = true; -++ #endif -+ cpuinfo_isa.fp16arith = true; -+ } -+ -+@@ -379,10 +378,22 @@ void cpuinfo_arm_mach_init(void) { -+ num_clusters * sizeof(struct cpuinfo_cluster), num_clusters); -+ goto cleanup; -+ } -++ uarchs = calloc(num_clusters, sizeof(struct cpuinfo_uarch_info)); -++ if (uarchs == NULL) { -++ cpuinfo_log_error( -++ "failed to allocate %zu bytes for descriptions of %"PRIu32" uarchs", -++ num_clusters * sizeof(enum cpuinfo_uarch), num_clusters); -++ goto cleanup; -++ } -+ uint32_t cluster_idx = UINT32_MAX; -+ for (uint32_t i = 0; i < mach_topology.cores; i++) { -+ if (i == 0 || cores[i].uarch != cores[i - 1].uarch) { -+ cluster_idx++; -++ uarchs[cluster_idx] = (struct cpuinfo_uarch_info) { -++ .uarch = cores[i].uarch, -++ .processor_count = 1, -++ .core_count = 1, -++ }; -+ clusters[cluster_idx] = (struct cpuinfo_cluster) { -+ .processor_start = i * threads_per_core, -+ .processor_count = 1, -+@@ -394,6 +405,8 @@ void cpuinfo_arm_mach_init(void) { -+ .uarch = cores[i].uarch, -+ }; -+ } else { -++ uarchs[cluster_idx].processor_count++; -++ uarchs[cluster_idx].core_count++; -+ clusters[cluster_idx].processor_count++; -+ clusters[cluster_idx].core_count++; -+ } -+@@ -542,26 +555,25 @@ void cpuinfo_arm_mach_init(void) { -+ } -+ -+ /* Commit changes */ -+- cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; -+- cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; -+- cpuinfo_cache[cpuinfo_cache_level_2] = l2; -+- cpuinfo_cache[cpuinfo_cache_level_3] = l3; -+- -+ cpuinfo_processors = processors; -+ cpuinfo_cores = cores; -+ cpuinfo_clusters = clusters; -+ cpuinfo_packages = packages; -+- -+- cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1_count; -+- cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1_count; -+- cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -+- cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -++ cpuinfo_uarchs = uarchs; -++ cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; -++ cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; -++ cpuinfo_cache[cpuinfo_cache_level_2] = l2; -++ cpuinfo_cache[cpuinfo_cache_level_3] = l3; -+ -+ cpuinfo_processors_count = mach_topology.threads; -+ cpuinfo_cores_count = mach_topology.cores; -+ cpuinfo_clusters_count = num_clusters; -+ cpuinfo_packages_count = mach_topology.packages; -+- -++ cpuinfo_uarchs_count = num_clusters; -++ cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1_count; -++ cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1_count; -++ cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -++ cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -+ cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); -+ -+ __sync_synchronize(); -+@@ -572,6 +584,7 @@ void cpuinfo_arm_mach_init(void) { -+ cores = NULL; -+ clusters = NULL; -+ packages = NULL; -++ uarchs = NULL; -+ l1i = l1d = l2 = l3 = NULL; -+ -+ cleanup: -+@@ -579,6 +592,7 @@ cleanup: -+ free(cores); -+ free(clusters); -+ free(packages); -++ free(uarchs); -+ free(l1i); -+ free(l1d); -+ free(l2); -+diff --git a/src/arm/uarch.c b/src/arm/uarch.c -+index a38250a..2aef9e7 100644 -+--- a/src/arm/uarch.c -++++ b/src/arm/uarch.c -+@@ -58,7 +58,9 @@ void cpuinfo_arm_decode_vendor_uarch( -+ *uarch = cpuinfo_uarch_cortex_a35; -+ break; -+ case 0xD05: -+- *uarch = cpuinfo_uarch_cortex_a55; -++ // Note: use Variant, not Revision, field -++ *uarch = (midr & CPUINFO_ARM_MIDR_VARIANT_MASK) == 0 ? -++ cpuinfo_uarch_cortex_a55r0 : cpuinfo_uarch_cortex_a55; -+ break; -+ case 0xD06: -+ *uarch = cpuinfo_uarch_cortex_a65; -+@@ -257,9 +259,9 @@ void cpuinfo_arm_decode_vendor_uarch( -+ *vendor = cpuinfo_vendor_arm; -+ *uarch = cpuinfo_uarch_cortex_a75; -+ break; -+- case 0x803: /* Low-power Kryo 385 "Silver" -> Cortex-A55 */ -++ case 0x803: /* Low-power Kryo 385 "Silver" -> Cortex-A55r0 */ -+ *vendor = cpuinfo_vendor_arm; -+- *uarch = cpuinfo_uarch_cortex_a55; -++ *uarch = cpuinfo_uarch_cortex_a55r0; -+ break; -+ case 0x804: /* High-performance Kryo 485 "Gold" / "Gold Prime" -> Cortex-A76 */ -+ *vendor = cpuinfo_vendor_arm; -+diff --git a/src/cpuinfo/common.h b/src/cpuinfo/common.h -+index 6ba746e..b2b404d 100644 -+--- a/src/cpuinfo/common.h -++++ b/src/cpuinfo/common.h -+@@ -12,29 +12,29 @@ -+ #define CPUINFO_COUNT_OF(array) (sizeof(array) / sizeof(0[array])) -+ -+ #if defined(__GNUC__) -+- #define CPUINFO_LIKELY(condition) (__builtin_expect(!!(condition), 1)) -+- #define CPUINFO_UNLIKELY(condition) (__builtin_expect(!!(condition), 0)) -++ #define CPUINFO_LIKELY(condition) (__builtin_expect(!!(condition), 1)) -++ #define CPUINFO_UNLIKELY(condition) (__builtin_expect(!!(condition), 0)) -+ #else -+- #define CPUINFO_LIKELY(condition) (!!(condition)) -+- #define CPUINFO_UNLIKELY(condition) (!!(condition)) -++ #define CPUINFO_LIKELY(condition) (!!(condition)) -++ #define CPUINFO_UNLIKELY(condition) (!!(condition)) -+ #endif -+ -+ #ifndef CPUINFO_INTERNAL -+- #if defined(__ELF__) -+- #define CPUINFO_INTERNAL __attribute__((__visibility__("internal"))) -+- #elif defined(__MACH__) -+- #define CPUINFO_INTERNAL __attribute__((__visibility__("hidden"))) -+- #else -+- #define CPUINFO_INTERNAL -+- #endif -++ #if defined(__ELF__) -++ #define CPUINFO_INTERNAL __attribute__((__visibility__("internal"))) -++ #elif defined(__MACH__) -++ #define CPUINFO_INTERNAL __attribute__((__visibility__("hidden"))) -++ #else -++ #define CPUINFO_INTERNAL -++ #endif -+ #endif -+ -+ #ifndef CPUINFO_PRIVATE -+- #if defined(__ELF__) -+- #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) -+- #elif defined(__MACH__) -+- #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) -+- #else -+- #define CPUINFO_PRIVATE -+- #endif -++ #if defined(__ELF__) -++ #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) -++ #elif defined(__MACH__) -++ #define CPUINFO_PRIVATE __attribute__((__visibility__("hidden"))) -++ #else -++ #define CPUINFO_PRIVATE -++ #endif -+ #endif -+diff --git a/src/cpuinfo/internal-api.h b/src/cpuinfo/internal-api.h -+index f12c48d..c6eed0b 100644 -+--- a/src/cpuinfo/internal-api.h -++++ b/src/cpuinfo/internal-api.h -+@@ -21,11 +21,13 @@ enum cpuinfo_cache_level { -+ }; -+ -+ extern CPUINFO_INTERNAL bool cpuinfo_is_initialized; -++ -+ extern CPUINFO_INTERNAL struct cpuinfo_processor* cpuinfo_processors; -+ extern CPUINFO_INTERNAL struct cpuinfo_core* cpuinfo_cores; -+ extern CPUINFO_INTERNAL struct cpuinfo_cluster* cpuinfo_clusters; -+ extern CPUINFO_INTERNAL struct cpuinfo_package* cpuinfo_packages; -+ extern CPUINFO_INTERNAL struct cpuinfo_cache* cpuinfo_cache[cpuinfo_cache_level_max]; -++ -+ extern CPUINFO_INTERNAL uint32_t cpuinfo_processors_count; -+ extern CPUINFO_INTERNAL uint32_t cpuinfo_cores_count; -+ extern CPUINFO_INTERNAL uint32_t cpuinfo_clusters_count; -+@@ -33,6 +35,19 @@ extern CPUINFO_INTERNAL uint32_t cpuinfo_packages_count; -+ extern CPUINFO_INTERNAL uint32_t cpuinfo_cache_count[cpuinfo_cache_level_max]; -+ extern CPUINFO_INTERNAL uint32_t cpuinfo_max_cache_size; -+ -++#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 -++ extern CPUINFO_INTERNAL struct cpuinfo_uarch_info* cpuinfo_uarchs; -++ extern CPUINFO_INTERNAL uint32_t cpuinfo_uarchs_count; -++#else -++ extern CPUINFO_INTERNAL struct cpuinfo_uarch_info cpuinfo_global_uarch; -++#endif -++ -++#ifdef __linux__ -++ extern CPUINFO_INTERNAL uint32_t cpuinfo_linux_cpu_max; -++ extern CPUINFO_INTERNAL const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map; -++ extern CPUINFO_INTERNAL const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map; -++#endif -++ -+ CPUINFO_PRIVATE void cpuinfo_x86_mach_init(void); -+ CPUINFO_PRIVATE void cpuinfo_x86_linux_init(void); -+ #ifdef _WIN32 -+diff --git a/src/linux/current.c b/src/linux/current.c -+deleted file mode 100644 -+index 472a4c9..0000000 -+--- a/src/linux/current.c -++++ /dev/null -+@@ -1,41 +0,0 @@ -+-#include -+-#include -+-#include -+-#include -+-#include -+- -+-#include -+- -+-#include -+-#include -+-#include -+-#include -+- -+- -+-const struct cpuinfo_processor** cpuinfo_linux_cpu_to_processor_map = NULL; -+-const struct cpuinfo_core** cpuinfo_linux_cpu_to_core_map = NULL; -+- -+- -+-const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void) { -+- if (!cpuinfo_is_initialized) { -+- cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_processor"); -+- } -+- const int cpu = sched_getcpu(); -+- if (cpu >= 0) { -+- return cpuinfo_linux_cpu_to_processor_map[cpu]; -+- } else { -+- return &cpuinfo_processors[0]; -+- } -+-} -+- -+-const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void) { -+- if (!cpuinfo_is_initialized) { -+- cpuinfo_log_fatal("cpuinfo_get_%s called before cpuinfo is initialized", "current_core"); -+- } -+- const int cpu = sched_getcpu(); -+- if (cpu >= 0) { -+- return cpuinfo_linux_cpu_to_core_map[cpu]; -+- } else { -+- return &cpuinfo_cores[0]; -+- } -+-} -+diff --git a/src/x86/api.h b/src/x86/api.h -+index 5f5e76d..213c2d8 100644 -+--- a/src/x86/api.h -++++ b/src/x86/api.h -+@@ -93,7 +93,6 @@ CPUINFO_INTERNAL struct cpuinfo_x86_isa cpuinfo_x86_detect_isa( -+ const struct cpuid_regs basic_info, const struct cpuid_regs extended_info, -+ uint32_t max_base_index, uint32_t max_extended_index, -+ enum cpuinfo_vendor vendor, enum cpuinfo_uarch uarch); -+-CPUINFO_INTERNAL struct cpuinfo_x86_isa cpuinfo_x86_nacl_detect_isa(void); -+ -+ CPUINFO_INTERNAL void cpuinfo_x86_detect_topology( -+ uint32_t max_base_index, -+diff --git a/src/x86/cache/init.c b/src/x86/cache/init.c -+index d581016..dd1f1ea 100644 -+--- a/src/x86/cache/init.c -++++ b/src/x86/cache/init.c -+@@ -65,7 +65,7 @@ iterate_descriptors: -+ } -+ } -+ -+- if (vendor != cpuinfo_vendor_amd && max_base_index >= 4) { -++ if (vendor != cpuinfo_vendor_amd && vendor != cpuinfo_vendor_hygon && max_base_index >= 4) { -+ struct cpuid_regs leaf4; -+ uint32_t input_ecx = 0; -+ uint32_t package_cores_max = 0; -+diff --git a/src/x86/cpuid.h b/src/x86/cpuid.h -+index 829ec21..9e9e013 100644 -+--- a/src/x86/cpuid.h -++++ b/src/x86/cpuid.h -+@@ -67,18 +67,13 @@ -+ } -+ #endif -+ -+-/* -+- * This instruction may be not supported by Native Client validator, -+- * make sure it doesn't appear in the binary -+- */ -+-#ifndef __native_client__ -+- static inline uint64_t xgetbv(uint32_t ext_ctrl_reg) { -+- #ifdef _MSC_VER -+- return (uint64_t)_xgetbv((unsigned int)ext_ctrl_reg); -+- #else -+- uint32_t lo, hi; -+- __asm__(".byte 0x0F, 0x01, 0xD0" : "=a" (lo), "=d" (hi) : "c" (ext_ctrl_reg)); -+- return ((uint64_t) hi << 32) | (uint64_t) lo; -+- #endif -+- } -+-#endif -++static inline uint64_t xgetbv(uint32_t ext_ctrl_reg) { -++ #ifdef _MSC_VER -++ return (uint64_t)_xgetbv((unsigned int)ext_ctrl_reg); -++ #else -++ uint32_t lo, hi; -++ __asm__(".byte 0x0F, 0x01, 0xD0" : "=a" (lo), "=d" (hi) : "c" (ext_ctrl_reg)); -++ return ((uint64_t) hi << 32) | (uint64_t) lo; -++ #endif -++} -++ -+diff --git a/src/x86/init.c b/src/x86/init.c -+index d736578..244359c 100644 -+--- a/src/x86/init.c -++++ b/src/x86/init.c -+@@ -61,12 +61,8 @@ void cpuinfo_x86_init_processor(struct cpuinfo_x86_processor* processor) { -+ -+ cpuinfo_x86_detect_topology(max_base_index, max_extended_index, leaf1, &processor->topology); -+ -+- #ifdef __native_client__ -+- cpuinfo_isa = cpuinfo_x86_nacl_detect_isa(); -+- #else -+- cpuinfo_isa = cpuinfo_x86_detect_isa(leaf1, leaf0x80000001, -+- max_base_index, max_extended_index, vendor, uarch); -+- #endif -++ cpuinfo_isa = cpuinfo_x86_detect_isa(leaf1, leaf0x80000001, -++ max_base_index, max_extended_index, vendor, uarch); -+ } -+ if (max_extended_index >= UINT32_C(0x80000004)) { -+ struct cpuid_regs brand_string[3]; -+diff --git a/src/x86/isa.c b/src/x86/isa.c -+index d27dbca..f2e5a28 100644 -+--- a/src/x86/isa.c -++++ b/src/x86/isa.c -+@@ -244,6 +244,7 @@ struct cpuinfo_x86_isa cpuinfo_x86_detect_isa( -+ */ -+ break; -+ case cpuinfo_vendor_amd: -++ case cpuinfo_vendor_hygon: -+ isa.prefetch = !!((extended_info.ecx & UINT32_C(0x00000100)) | (extended_info.edx & UINT32_C(0xE0000000))); -+ break; -+ default: -+@@ -265,6 +266,7 @@ struct cpuinfo_x86_isa cpuinfo_x86_detect_isa( -+ */ -+ switch (vendor) { -+ case cpuinfo_vendor_amd: -++ case cpuinfo_vendor_hygon: -+ isa.prefetchw = !!((extended_info.ecx & UINT32_C(0x00000100)) | (extended_info.edx & UINT32_C(0xE0000000))); -+ break; -+ default: -+diff --git a/src/x86/linux/init.c b/src/x86/linux/init.c -+index c096336..f565789 100644 -+--- a/src/x86/linux/init.c -++++ b/src/x86/linux/init.c -+@@ -569,9 +569,6 @@ void cpuinfo_x86_linux_init(void) { -+ } -+ -+ /* Commit changes */ -+- cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; -+- cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; -+- -+ cpuinfo_processors = processors; -+ cpuinfo_cores = cores; -+ cpuinfo_clusters = clusters; -+@@ -591,24 +588,32 @@ void cpuinfo_x86_linux_init(void) { -+ cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; -+- -+ cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); -+ -++ cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { -++ .uarch = x86_processor.uarch, -++ .cpuid = x86_processor.cpuid, -++ .processor_count = processors_count, -++ .core_count = cores_count, -++ }; -++ -++ cpuinfo_linux_cpu_max = x86_linux_processors_count; -++ cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map; -++ cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map; -++ -+ __sync_synchronize(); -+ -+ cpuinfo_is_initialized = true; -+ -+- linux_cpu_to_processor_map = NULL; -+- linux_cpu_to_core_map = NULL; -+ processors = NULL; -+ cores = NULL; -+ clusters = NULL; -+ packages = NULL; -+ l1i = l1d = l2 = l3 = l4 = NULL; -++ linux_cpu_to_processor_map = NULL; -++ linux_cpu_to_core_map = NULL; -+ -+ cleanup: -+- free(linux_cpu_to_processor_map); -+- free(linux_cpu_to_core_map); -+ free(x86_linux_processors); -+ free(processors); -+ free(cores); -+@@ -619,4 +624,6 @@ cleanup: -+ free(l2); -+ free(l3); -+ free(l4); -++ free(linux_cpu_to_processor_map); -++ free(linux_cpu_to_core_map); -+ } -+diff --git a/src/x86/mach/init.c b/src/x86/mach/init.c -+index ae2be33..b44d3ad 100644 -+--- a/src/x86/mach/init.c -++++ b/src/x86/mach/init.c -+@@ -305,30 +305,34 @@ void cpuinfo_x86_mach_init(void) { -+ } -+ -+ /* Commit changes */ -++ cpuinfo_processors = processors; -++ cpuinfo_cores = cores; -++ cpuinfo_clusters = clusters; -++ cpuinfo_packages = packages; -+ cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; -+ cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; -+ cpuinfo_cache[cpuinfo_cache_level_2] = l2; -+ cpuinfo_cache[cpuinfo_cache_level_3] = l3; -+ cpuinfo_cache[cpuinfo_cache_level_4] = l4; -+ -+- cpuinfo_processors = processors; -+- cpuinfo_cores = cores; -+- cpuinfo_clusters = clusters; -+- cpuinfo_packages = packages; -+- -++ cpuinfo_processors_count = mach_topology.threads; -++ cpuinfo_cores_count = mach_topology.cores; -++ cpuinfo_clusters_count = mach_topology.packages; -++ cpuinfo_packages_count = mach_topology.packages; -+ cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; -+- -+- cpuinfo_processors_count = mach_topology.threads; -+- cpuinfo_cores_count = mach_topology.cores; -+- cpuinfo_clusters_count = mach_topology.packages; -+- cpuinfo_packages_count = mach_topology.packages; -+- -+ cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); -+ -++ cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { -++ .uarch = x86_processor.uarch, -++ .cpuid = x86_processor.cpuid, -++ .processor_count = mach_topology.threads, -++ .core_count = mach_topology.cores, -++ }; -++ -+ __sync_synchronize(); -+ -+ cpuinfo_is_initialized = true; -+diff --git a/src/x86/nacl/isa.c b/src/x86/nacl/isa.c -+deleted file mode 100644 -+index 662be33..0000000 -+--- a/src/x86/nacl/isa.c -++++ /dev/null -+@@ -1,306 +0,0 @@ -+-#include -+-#include -+-#include -+- -+-#include -+- -+-#define NACL_CODE_BUNDLE_SIZE 32 -+-#include -+-#include -+- -+-static const uint8_t cmpxchg16b_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* MOV edi, edi */ -+- 0x89, 0xFF, -+- /* CMPXCHG16B [r15 + rdi * 1] */ -+- 0x49, 0x0F, 0xC7, 0x0C, 0x3F, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t lzcnt_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* LZCNT eax, ecx */ -+- 0xF3, 0x0F, 0xBD, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t popcnt_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* POPCNT eax, ecx */ -+- 0xF3, 0x0F, 0xB8, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t movbe_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* MOV ecx, ecx */ -+- 0x89, 0xC9, -+- /* MOVBE eax, [r15 + rcx * 1] */ -+- 0x41, 0x0F, 0x38, 0xF0, 0x04, 0x0F, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t bmi_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* ANDN eax, ecx, edx */ -+- 0xC4, 0xE2, 0x70, 0xF2, 0xC2, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t tbm_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* BLCS eax, ecx */ -+- 0x8F, 0xE9, 0x78, 0x01, 0xD9, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t three_d_now_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PFADD mm0, mm1 */ -+- 0x0F, 0x0F, 0xC1, 0x9E, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t three_d_now_plus_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PFNACC mm0, mm1 */ -+- 0x0F, 0x0F, 0xC1, 0x8A, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t sse3_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* HADDPS xmm0, xmm1 */ -+- 0xF2, 0x0F, 0x7C, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t ssse3_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PSHUFB xmm0, xmm1 */ -+- 0x66, 0x0F, 0x38, 0x00, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t sse4_1_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PMULLD xmm0, xmm1 */ -+- 0x66, 0x0F, 0x38, 0x40, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t sse4_2_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PCMPGTQ xmm0, xmm1 */ -+- 0x66, 0x0F, 0x38, 0x37, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t sse4a_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* EXTRQ xmm0, xmm1 */ -+- 0x66, 0x0F, 0x79, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t aes_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* AESENC xmm0, xmm1 */ -+- 0x66, 0x0F, 0x38, 0xDC, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t pclmulqdq_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* PCLMULQDQ xmm0, xmm1, 0 */ -+- 0x66, 0x0F, 0x3A, 0x44, 0xC1, 0x00, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t avx_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VPERMILPS ymm0, ymm1, 0xAA */ -+- 0xC4, 0xE3, 0x7D, 0x04, 0xC1, 0xAA, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t fma3_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VFMADDSUB213PS ymm0, ymm1, ymm2 */ -+- 0xC4, 0xE2, 0x75, 0xA6, 0xC2, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t fma4_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VFMADDPS ymm0, ymm1, ymm2, ymm3 */ -+- 0xC4, 0xE3, 0xF5, 0x68, 0xC3, 0x20, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t xop_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VPHADDBQ xmm0, xmm1 */ -+- 0x8F, 0xE9, 0x78, 0xC3, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t f16c_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VCVTPH2PS ymm0, xmm1 */ -+- 0xC4, 0xE2, 0x7D, 0x13, 0xC1, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+-static const uint8_t avx2_bundle[NACL_CODE_BUNDLE_SIZE] = { -+- /* VPERMPS ymm0, ymm1, ymm2 */ -+- 0xC4, 0xE2, 0x75, 0x16, 0xC2, -+- /* Fill remainder with HLTs */ -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+- 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, 0xF4, -+-}; -+- -+- -+-struct cpuinfo_x86_isa cpuinfo_x86_nacl_detect_isa(void) { -+- /* -+- * Under Native Client sandbox we can't just ask the CPU: -+- * - First, some instructions (XGETBV) necessary to query AVX support are not white-listed in the validator. -+- * - Secondly, even if CPU supports some instruction, but validator doesn't know about it (e.g. due a bug in the -+- * ISA detection in the validator), all instructions from the "unsupported" ISA extensions will be replaced by -+- * HLTs when the module is loaded. -+- * Thus, instead of quering the CPU about supported ISA extensions, we query the validator: we pass bundles with -+- * instructions from ISA extensions to dynamic code generation APIs, and test if they are accepted. -+- */ -+- -+- struct cpuinfo_x86_isa isa = { 0 }; -+- -+- struct nacl_irt_code_data_alloc nacl_irt_code_data_alloc = { 0 }; -+- struct nacl_irt_dyncode nacl_irt_dyncode = { 0 }; -+- if (sizeof(nacl_irt_code_data_alloc) != nacl_interface_query(NACL_IRT_CODE_DATA_ALLOC_v0_1, -+- &nacl_irt_code_data_alloc, -+- sizeof(nacl_irt_code_data_alloc))) -+- { -+- goto finish; -+- } -+- -+- if (sizeof(nacl_irt_dyncode) != nacl_interface_query(NACL_IRT_DYNCODE_v0_1, -+- &nacl_irt_dyncode, -+- sizeof(nacl_irt_dyncode))) -+- { -+- goto finish; -+- } -+- -+- const size_t allocation_size = 65536; -+- uintptr_t code_segment = 0; -+- if (0 != nacl_irt_code_data_alloc.allocate_code_data(0, allocation_size, 0, 0, &code_segment)) -+- { -+- goto finish; -+- } -+- -+- isa.cmpxchg16b = !nacl_irt_dyncode.dyncode_create((void*) code_segment, cmpxchg16b_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.lzcnt = !nacl_irt_dyncode.dyncode_create((void*) code_segment, lzcnt_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.popcnt = !nacl_irt_dyncode.dyncode_create((void*) code_segment, popcnt_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.movbe = !nacl_irt_dyncode.dyncode_create((void*) code_segment, movbe_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.bmi = !nacl_irt_dyncode.dyncode_create((void*) code_segment, bmi_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.tbm = !nacl_irt_dyncode.dyncode_create((void*) code_segment, tbm_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.three_d_now = !nacl_irt_dyncode.dyncode_create((void*) code_segment, three_d_now_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.three_d_now_plus = -+- !nacl_irt_dyncode.dyncode_create((void*) code_segment, three_d_now_plus_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.sse3 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, sse3_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.ssse3 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, ssse3_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.sse4_1 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, sse4_1_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.sse4_2 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, sse4_2_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.sse4a = !nacl_irt_dyncode.dyncode_create((void*) code_segment, sse4a_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.aes = !nacl_irt_dyncode.dyncode_create((void*) code_segment, aes_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.pclmulqdq = !nacl_irt_dyncode.dyncode_create((void*) code_segment, pclmulqdq_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.avx = !nacl_irt_dyncode.dyncode_create((void*) code_segment, avx_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.fma3 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, fma3_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.fma4 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, fma4_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.xop = !nacl_irt_dyncode.dyncode_create((void*) code_segment, xop_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.f16c = !nacl_irt_dyncode.dyncode_create((void*) code_segment, f16c_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- code_segment += NACL_CODE_BUNDLE_SIZE; -+- -+- isa.avx2 = !nacl_irt_dyncode.dyncode_create((void*) code_segment, avx2_bundle, NACL_CODE_BUNDLE_SIZE) && -+- (*((const uint8_t*) code_segment) != 0xF4); -+- -+-finish: -+- return isa; -+-} -+diff --git a/src/x86/name.c b/src/x86/name.c -+index 708be1d..e0d5a5b 100644 -+--- a/src/x86/name.c -++++ b/src/x86/name.c -+@@ -671,6 +671,7 @@ static const char* vendor_string_map[] = { -+ [cpuinfo_vendor_intel] = "Intel", -+ [cpuinfo_vendor_amd] = "AMD", -+ [cpuinfo_vendor_via] = "VIA", -++ [cpuinfo_vendor_hygon] = "Hygon", -+ [cpuinfo_vendor_rdc] = "RDC", -+ [cpuinfo_vendor_dmp] = "DM&P", -+ [cpuinfo_vendor_transmeta] = "Transmeta", -+diff --git a/src/x86/uarch.c b/src/x86/uarch.c -+index ba72d8a..ecaa762 100644 -+--- a/src/x86/uarch.c -++++ b/src/x86/uarch.c -+@@ -79,6 +79,8 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch( -+ case 0x5E: // Sky Lake Client DT/H/S -+ case 0x8E: // Kaby/Whiskey/Amber/Comet Lake Y/U -+ case 0x9E: // Kaby/Coffee Lake DT/H/S -++ case 0xA5: // Comet Lake H/S -++ case 0xA6: // Comet Lake U/Y -+ return cpuinfo_uarch_sky_lake; -+ case 0x66: // Cannon Lake (Core i3-8121U) -+ return cpuinfo_uarch_palm_cove; -+@@ -94,7 +96,7 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch( -+ return cpuinfo_uarch_bonnell; -+ case 0x27: // Medfield -+ case 0x35: // Cloverview -+- case 0x36: // Cedarview, Centerton -++ case 0x36: // Cedarview, Centerton -+ return cpuinfo_uarch_saltwell; -+ case 0x37: // Bay Trail -+ case 0x4A: // Merrifield -+@@ -110,6 +112,7 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch( -+ return cpuinfo_uarch_goldmont; -+ case 0x7A: // Gemini Lake -+ return cpuinfo_uarch_goldmont_plus; -++ -+ /* Knights-series cores */ -+ case 0x57: -+ return cpuinfo_uarch_knights_landing; -+@@ -173,7 +176,7 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch( -+ case 0x38: // Godavari -+ case 0x30: // Kaveri -+ return cpuinfo_uarch_steamroller; -+- case 0x60: // Carrizo -++ case 0x60: // Carrizo -+ case 0x65: // Bristol Ridge -+ case 0x70: // Stoney Ridge -+ return cpuinfo_uarch_excavator; -+@@ -201,14 +204,22 @@ enum cpuinfo_uarch cpuinfo_x86_decode_uarch( -+ switch (model_info->model) { -+ case 0x01: // 14 nm Naples, Whitehaven, Summit Ridge, Snowy Owl -+ case 0x08: // 12 nm Pinnacle Ridge -+- case 0x11: // 14 nm Raven Ridge -++ case 0x11: // 14 nm Raven Ridge, Great Horned Owl -+ case 0x18: // 12 nm Picasso -+ return cpuinfo_uarch_zen; -++ case 0x31: // Rome, Castle Peak -++ case 0x60: // Renoir -+ case 0x71: // Matisse -+ return cpuinfo_uarch_zen2; -+ } -+ } -+ break; -++ case cpuinfo_vendor_hygon: -++ switch (model_info->family) { -++ case 0x00: -++ return cpuinfo_uarch_dhyana; -++ } -++ break; -+ default: -+ break; -+ } -+diff --git a/src/x86/vendor.c b/src/x86/vendor.c -+index 3f3c753..2bba90d 100644 -+--- a/src/x86/vendor.c -++++ b/src/x86/vendor.c -+@@ -26,6 +26,11 @@ -+ #define auls UINT32_C(0x736C7561) -+ #define VIA UINT32_C(0x20414956) -+ -++/* Hygon vendor string: "HygonGenuine" */ -++#define Hygo UINT32_C(0x6F677948) -++#define nGen UINT32_C(0x6E65476E) -++#define uine UINT32_C(0x656E6975) -++ -+ /* Transmeta vendor strings: "GenuineTMx86", "TransmetaCPU" */ -+ #define ineT UINT32_C(0x54656E69) -+ #define Mx86 UINT32_C(0x3638784D) -+@@ -105,6 +110,12 @@ enum cpuinfo_vendor cpuinfo_x86_decode_vendor(uint32_t ebx, uint32_t ecx, uint32 -+ return cpuinfo_vendor_via; -+ } -+ break; -++ case Hygo: -++ if (edx == nGen && ecx == uine) { -++ /* "HygonGenuine" */ -++ return cpuinfo_vendor_hygon; -++ } -++ break; -+ #if CPUINFO_ARCH_X86 -+ case AMDi: -+ if (edx == sbet && ecx == ter) { -+diff --git a/src/x86/windows/init.c b/src/x86/windows/init.c -+index 7a2090e..2c7e3cd 100644 -+--- a/src/x86/windows/init.c -++++ b/src/x86/windows/init.c -+@@ -417,9 +417,6 @@ BOOL CALLBACK cpuinfo_x86_windows_init(PINIT_ONCE init_once, PVOID parameter, PV -+ for (uint32_t i = 0; i < processors_count; i++) { -+ const uint32_t apic_id = processors[i].apic_id; -+ -+- //linux_cpu_to_processor_map[x86_linux_processors[i].linux_id] = processors + processor_index; -+- //linux_cpu_to_core_map[x86_linux_processors[i].linux_id] = cores + core_index; -+- -+ if (x86_processor.cache.l1i.size != 0) { -+ const uint32_t l1i_id = apic_id & ~bit_mask(x86_processor.cache.l1i.apic_bits); -+ processors[i].cache.l1i = &l1i[l1i_index]; -+@@ -549,30 +546,34 @@ BOOL CALLBACK cpuinfo_x86_windows_init(PINIT_ONCE init_once, PVOID parameter, PV -+ -+ -+ /* Commit changes */ -++ cpuinfo_processors = processors; -++ cpuinfo_cores = cores; -++ cpuinfo_clusters = clusters; -++ cpuinfo_packages = packages; -+ cpuinfo_cache[cpuinfo_cache_level_1i] = l1i; -+ cpuinfo_cache[cpuinfo_cache_level_1d] = l1d; -+ cpuinfo_cache[cpuinfo_cache_level_2] = l2; -+ cpuinfo_cache[cpuinfo_cache_level_3] = l3; -+ cpuinfo_cache[cpuinfo_cache_level_4] = l4; -+ -+- cpuinfo_processors = processors; -+- cpuinfo_cores = cores; -+- cpuinfo_clusters = clusters; -+- cpuinfo_packages = packages; -+- -++ cpuinfo_processors_count = processors_count; -++ cpuinfo_cores_count = cores_count; -++ cpuinfo_clusters_count = packages_count; -++ cpuinfo_packages_count = packages_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_1i] = l1i_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_1d] = l1d_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count; -+ cpuinfo_cache_count[cpuinfo_cache_level_4] = l4_count; -+- -+- cpuinfo_processors_count = processors_count; -+- cpuinfo_cores_count = cores_count; -+- cpuinfo_clusters_count = packages_count; -+- cpuinfo_packages_count = packages_count; -+- -+ cpuinfo_max_cache_size = cpuinfo_compute_max_cache_size(&processors[0]); -+ -++ cpuinfo_global_uarch = (struct cpuinfo_uarch_info) { -++ .uarch = x86_processor.uarch, -++ .cpuid = x86_processor.cpuid, -++ .processor_count = processors_count, -++ .core_count = cores_count, -++ }; -++ -+ MemoryBarrier(); -+ -+ cpuinfo_is_initialized = true; -+diff --git a/test/arm-cache.cc b/test/arm-cache.cc -+index 8373f7c..7d2e4a4 100644 -+--- a/test/arm-cache.cc -++++ b/test/arm-cache.cc -+@@ -766,7 +766,7 @@ TEST(QUALCOMM, snapdragon_845) { -+ struct cpuinfo_cache little_l2 = { 0 }; -+ struct cpuinfo_cache little_l3 = { 0 }; -+ cpuinfo_arm_decode_cache( -+- cpuinfo_uarch_cortex_a55, 4, UINT32_C(0x518F803C), -++ cpuinfo_uarch_cortex_a55r0, 4, UINT32_C(0x518F803C), -+ &chipset, 1, 8, -+ &little_l1i, &little_l1d, &little_l2, &little_l3); -+ -+@@ -910,7 +910,7 @@ TEST(SAMSUNG, exynos_9810) { -+ struct cpuinfo_cache little_l2 = { 0 }; -+ struct cpuinfo_cache little_l3 = { 0 }; -+ cpuinfo_arm_decode_cache( -+- cpuinfo_uarch_cortex_a55, 4, UINT32_C(0x410FD051), -++ cpuinfo_uarch_cortex_a55r0, 4, UINT32_C(0x410FD051), -+ &chipset, 1, 8, -+ &little_l1i, &little_l1d, &little_l2, &little_l3); -+ -+diff --git a/test/get-current.cc b/test/get-current.cc -+index 4a80cab..f410b12 100644 -+--- a/test/get-current.cc -++++ b/test/get-current.cc -+@@ -3,34 +3,36 @@ -+ #include -+ -+ -+-TEST(CURRENT_PROCESSOR, not_null) { -+- ASSERT_TRUE(cpuinfo_initialize()); -+- -+- ASSERT_TRUE(cpuinfo_get_current_processor()); -+-} -+- -+ TEST(CURRENT_PROCESSOR, within_bounds) { -+ ASSERT_TRUE(cpuinfo_initialize()); -+ -+ const struct cpuinfo_processor* current_processor = cpuinfo_get_current_processor(); -++ if (current_processor == nullptr) { -++ GTEST_SKIP(); -++ } -++ -+ const struct cpuinfo_processor* processors_begin = cpuinfo_get_processors(); -+ const struct cpuinfo_processor* processors_end = processors_begin + cpuinfo_get_processors_count(); -+ ASSERT_GE(current_processor, processors_begin); -+ ASSERT_LT(current_processor, processors_end); -+ } -+ -+-TEST(CURRENT_CORE, not_null) { -+- ASSERT_TRUE(cpuinfo_initialize()); -+- -+- ASSERT_TRUE(cpuinfo_get_current_core()); -+-} -+- -+ TEST(CURRENT_CORE, within_bounds) { -+ ASSERT_TRUE(cpuinfo_initialize()); -+ -+ const struct cpuinfo_core* current_core = cpuinfo_get_current_core(); -++ if (current_core == nullptr) { -++ GTEST_SKIP(); -++ } -++ -+ const struct cpuinfo_core* cores_begin = cpuinfo_get_cores(); -+ const struct cpuinfo_core* cores_end = cores_begin + cpuinfo_get_cores_count(); -+ ASSERT_GE(current_core, cores_begin); -+ ASSERT_LT(current_core, cores_end); -+ } -++ -++TEST(CURRENT_UARCH_INDEX, within_bounds) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ -++ ASSERT_LT(cpuinfo_get_current_uarch_index(), cpuinfo_get_uarchs_count()); -++} -+diff --git a/test/init.cc b/test/init.cc -+index 941cb97..718eb96 100644 -+--- a/test/init.cc -++++ b/test/init.cc -+@@ -678,6 +678,72 @@ TEST(PACKAGE, consistent_cluster) { -+ cpuinfo_deinitialize(); -+ } -+ -++TEST(UARCHS_COUNT, within_bounds) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ EXPECT_NE(0, cpuinfo_get_uarchs_count()); -++ EXPECT_LE(cpuinfo_get_packages_count(), cpuinfo_get_cores_count()); -++ EXPECT_LE(cpuinfo_get_packages_count(), cpuinfo_get_processors_count()); -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCHS, non_null) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ EXPECT_TRUE(cpuinfo_get_uarchs()); -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCH, non_null) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ EXPECT_TRUE(cpuinfo_get_uarch(i)); -++ } -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCH, non_zero_processors) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ const cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(i); -++ ASSERT_TRUE(uarch); -++ -++ EXPECT_NE(0, uarch->processor_count); -++ } -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCH, valid_processors) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ const cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(i); -++ ASSERT_TRUE(uarch); -++ -++ EXPECT_LE(uarch->processor_count, cpuinfo_get_processors_count()); -++ } -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCH, non_zero_cores) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ const cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(i); -++ ASSERT_TRUE(uarch); -++ -++ EXPECT_NE(0, uarch->core_count); -++ } -++ cpuinfo_deinitialize(); -++} -++ -++TEST(UARCH, valid_cores) { -++ ASSERT_TRUE(cpuinfo_initialize()); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ const cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(i); -++ ASSERT_TRUE(uarch); -++ -++ EXPECT_LE(uarch->core_count, cpuinfo_get_cores_count()); -++ } -++ cpuinfo_deinitialize(); -++} -++ -+ TEST(L1I_CACHES_COUNT, within_bounds) { -+ ASSERT_TRUE(cpuinfo_initialize()); -+ EXPECT_NE(0, cpuinfo_get_l1i_caches_count()); -+diff --git a/test/mock/galaxy-s9-global.cc b/test/mock/galaxy-s9-global.cc -+index 7a67129..6c72513 100644 -+--- a/test/mock/galaxy-s9-global.cc -++++ b/test/mock/galaxy-s9-global.cc -+@@ -207,7 +207,7 @@ TEST(CORES, uarch) { -+ case 5: -+ case 6: -+ case 7: -+- ASSERT_EQ(cpuinfo_uarch_cortex_a55, cpuinfo_get_core(i)->uarch); -++ ASSERT_EQ(cpuinfo_uarch_cortex_a55r0, cpuinfo_get_core(i)->uarch); -+ break; -+ } -+ } -+@@ -329,7 +329,7 @@ TEST(CLUSTERS, uarch) { -+ ASSERT_EQ(cpuinfo_uarch_exynos_m3, cpuinfo_get_cluster(i)->uarch); -+ break; -+ case 1: -+- ASSERT_EQ(cpuinfo_uarch_cortex_a55, cpuinfo_get_cluster(i)->uarch); -++ ASSERT_EQ(cpuinfo_uarch_cortex_a55r0, cpuinfo_get_cluster(i)->uarch); -+ break; -+ } -+ } -+diff --git a/test/mock/galaxy-s9-us.cc b/test/mock/galaxy-s9-us.cc -+index 6df7f3c..ceea969 100644 -+--- a/test/mock/galaxy-s9-us.cc -++++ b/test/mock/galaxy-s9-us.cc -+@@ -168,7 +168,7 @@ TEST(CORES, uarch) { -+ case 5: -+ case 6: -+ case 7: -+- ASSERT_EQ(cpuinfo_uarch_cortex_a55, cpuinfo_get_core(i)->uarch); -++ ASSERT_EQ(cpuinfo_uarch_cortex_a55r0, cpuinfo_get_core(i)->uarch); -+ break; -+ } -+ } -+@@ -283,7 +283,7 @@ TEST(CLUSTERS, uarch) { -+ ASSERT_EQ(cpuinfo_uarch_cortex_a75, cpuinfo_get_cluster(i)->uarch); -+ break; -+ case 1: -+- ASSERT_EQ(cpuinfo_uarch_cortex_a55, cpuinfo_get_cluster(i)->uarch); -++ ASSERT_EQ(cpuinfo_uarch_cortex_a55r0, cpuinfo_get_cluster(i)->uarch); -+ break; -+ } -+ } -+@@ -817,4 +817,4 @@ int main(int argc, char* argv[]) { -+ cpuinfo_initialize(); -+ ::testing::InitGoogleTest(&argc, argv); -+ return RUN_ALL_TESTS(); -+-} -+\ No newline at end of file -++} -+diff --git a/tools/cpu-info.c b/tools/cpu-info.c -+index 7fa5187..7963c00 100644 -+--- a/tools/cpu-info.c -++++ b/tools/cpu-info.c -+@@ -14,6 +14,8 @@ static const char* vendor_to_string(enum cpuinfo_vendor vendor) { -+ return "Intel"; -+ case cpuinfo_vendor_amd: -+ return "AMD"; -++ case cpuinfo_vendor_hygon: -++ return "Hygon"; -+ case cpuinfo_vendor_arm: -+ return "ARM"; -+ case cpuinfo_vendor_qualcomm: -+@@ -161,6 +163,8 @@ static const char* uarch_to_string(enum cpuinfo_uarch uarch) { -+ return "Cortex-A35"; -+ case cpuinfo_uarch_cortex_a53: -+ return "Cortex-A53"; -++ case cpuinfo_uarch_cortex_a55r0: -++ return "Cortex-A55r0"; -+ case cpuinfo_uarch_cortex_a55: -+ return "Cortex-A55"; -+ case cpuinfo_uarch_cortex_a57: -+@@ -223,6 +227,10 @@ static const char* uarch_to_string(enum cpuinfo_uarch uarch) { -+ return "Vortex"; -+ case cpuinfo_uarch_tempest: -+ return "Tempest"; -++ case cpuinfo_uarch_lightning: -++ return "Lightning"; -++ case cpuinfo_uarch_thunder: -++ return "Thunder"; -+ case cpuinfo_uarch_thunderx: -+ return "ThunderX"; -+ case cpuinfo_uarch_thunderx2: -+@@ -253,6 +261,17 @@ int main(int argc, char** argv) { -+ printf("\t%"PRIu32": %s\n", i, cpuinfo_get_package(i)->name); -+ } -+ #endif -++ printf("Microarchitectures:\n"); -++ for (uint32_t i = 0; i < cpuinfo_get_uarchs_count(); i++) { -++ const struct cpuinfo_uarch_info* uarch_info = cpuinfo_get_uarch(i); -++ const char* uarch_string = uarch_to_string(uarch_info->uarch); -++ if (uarch_string == NULL) { -++ printf("\t%"PRIu32"x Unknown (0x%08"PRIx32"\n", -++ uarch_info->core_count, (uint32_t) uarch_info->uarch); -++ } else { -++ printf("\t%"PRIu32"x %s\n", uarch_info->core_count, uarch_string); -++ } -++ } -+ printf("Cores:\n"); -+ for (uint32_t i = 0; i < cpuinfo_get_cores_count(); i++) { -+ const struct cpuinfo_core* core = cpuinfo_get_core(i); -+@@ -277,17 +296,17 @@ int main(int argc, char** argv) { -+ } -+ } -+ printf("Logical processors"); -+- #if defined(__linux__) -+- printf(" (System ID)"); -+- #endif -+- printf(":\n"); -++ #if defined(__linux__) -++ printf(" (System ID)"); -++ #endif -++ printf(":\n"); -+ for (uint32_t i = 0; i < cpuinfo_get_processors_count(); i++) { -+ const struct cpuinfo_processor* processor = cpuinfo_get_processor(i); -+- printf("\t%"PRIu32"", i); -++ printf("\t%"PRIu32"", i); -+ -+- #if defined(__linux__) -+- printf(" (%"PRId32")", processor->linux_id); -+- #endif -++ #if defined(__linux__) -++ printf(" (%"PRId32")", processor->linux_id); -++ #endif -+ -+ #if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 -+ printf(": APIC ID 0x%08"PRIx32"\n", processor->apic_id); -diff --git a/third_party/cpuinfo/workspace.bzl b/third_party/cpuinfo/workspace.bzl -index c2eeede8a0..77aecf5a9a 100644 ---- a/third_party/cpuinfo/workspace.bzl -+++ b/third_party/cpuinfo/workspace.bzl -@@ -2,14 +2,20 @@ - - load("//third_party:repo.bzl", "third_party_http_archive") - -+# Sanitize a dependency so that it works correctly from code that includes -+# TensorFlow as a submodule. -+def clean_dep(dep): -+ return str(Label(dep)) -+ - def repo(): - third_party_http_archive( - name = "cpuinfo", -- strip_prefix = "cpuinfo-e39a5790059b6b8274ed91f7b5b5b13641dff267", -- sha256 = "e5caa8b7c58f1623eed88f4d5147e3753ff19cde821526bc9aa551b004f751fe", -+ strip_prefix = "cpuinfo-d6c0f915ee737f961915c9d17f1679b6777af207", -+ sha256 = "146fc61c3cf63d7d88db963876929a4d373f621fb65568b895efa0857f467770", - urls = [ -- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pytorch/cpuinfo/archive/e39a5790059b6b8274ed91f7b5b5b13641dff267.tar.gz", -- "https://github.com/pytorch/cpuinfo/archive/e39a5790059b6b8274ed91f7b5b5b13641dff267.tar.gz", -+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pytorch/cpuinfo/archive/d6c0f915ee737f961915c9d17f1679b6777af207.tar.gz", -+ "https://github.com/pytorch/cpuinfo/archive/d6c0f915ee737f961915c9d17f1679b6777af207.tar.gz", - ], - build_file = "//third_party/cpuinfo:BUILD.bazel", -+ patch_file = clean_dep("//third_party/cpuinfo:cpuinfo.patch"), - ) diff --git a/third_party/org_tensorflow_compatibility_fixes.diff b/third_party/org_tensorflow_compatibility_fixes.diff new file mode 100644 index 000000000..4f8579d98 --- /dev/null +++ b/third_party/org_tensorflow_compatibility_fixes.diff @@ -0,0 +1,25 @@ +diff --git a/tensorflow/core/lib/monitoring/percentile_sampler.cc b/tensorflow/core/lib/monitoring/percentile_sampler.cc +index b7c22ae77b..d0ba7b48b4 100644 +--- a/tensorflow/core/lib/monitoring/percentile_sampler.cc ++++ b/tensorflow/core/lib/monitoring/percentile_sampler.cc +@@ -29,7 +29,8 @@ namespace monitoring { + void PercentileSamplerCell::Add(double sample) { + uint64 nstime = EnvTime::NowNanos(); + mutex_lock l(mu_); +- samples_[next_position_] = {nstime, sample}; ++ samples_[next_position_].nstime = nstime; ++ samples_[next_position_].value = sample; + ++next_position_; + if (TF_PREDICT_FALSE(next_position_ >= samples_.size())) { + next_position_ = 0; +@@ -73,7 +74,9 @@ Percentiles PercentileSamplerCell::value() const { + size_t index = std::min( + static_cast(percentile * pct_samples.num_samples / 100.0), + pct_samples.num_samples - 1); +- PercentilePoint pct = {percentile, samples[index].value}; ++ PercentilePoint pct; ++ pct.percentile = percentile; ++ pct.value = samples[index].value; + pct_samples.points.push_back(pct); + } + } diff --git a/third_party/org_tensorflow_protobuf_updates.diff b/third_party/org_tensorflow_protobuf_updates.diff new file mode 100644 index 000000000..39ae6d61f --- /dev/null +++ b/third_party/org_tensorflow_protobuf_updates.diff @@ -0,0 +1,90 @@ +diff --git a/third_party/protobuf/protobuf.patch b/third_party/protobuf/protobuf.patch +index 8ce4a84375..d77e1f81f3 100644 +--- a/third_party/protobuf/protobuf.patch ++++ b/third_party/protobuf/protobuf.patch +@@ -1,25 +1,25 @@ + diff --git a/BUILD b/BUILD +-index dbae719ff..87dc38470 100644 ++index 79871d621..096967b99 100644 + --- a/BUILD + +++ b/BUILD +-@@ -23,7 +23,7 @@ config_setting( ++@@ -26,7 +26,7 @@ config_setting( + # ZLIB configuration + ################################################################################ +- ++ + -ZLIB_DEPS = ["@zlib//:zlib"] + +ZLIB_DEPS = ["@zlib"] +- ++ + ################################################################################ + # Protobuf Runtime Library +-@@ -143,6 +143,7 @@ cc_library( ++@@ -156,6 +156,7 @@ cc_library( + copts = COPTS, + includes = ["src/"], + linkopts = LINK_OPTS, + + alwayslink = 1, + visibility = ["//visibility:public"], + ) +- +-@@ -213,6 +214,7 @@ cc_library( ++ ++@@ -228,6 +229,7 @@ cc_library( + copts = COPTS, + includes = ["src/"], + linkopts = LINK_OPTS, +@@ -27,17 +27,24 @@ index dbae719ff..87dc38470 100644 + visibility = ["//visibility:public"], + deps = [":protobuf_lite"] + PROTOBUF_DEPS, + ) ++@@ -900,7 +902,6 @@ py_proto_library( ++ py_extra_srcs = glob(["python/**/__init__.py"]), ++ py_libs = [ ++ ":python_srcs", ++- "@six//:six", ++ ], ++ srcs_version = "PY2AND3", ++ visibility = ["//visibility:public"], + diff --git a/protobuf.bzl b/protobuf.bzl +-index e0653321f..253d9cbb5 100644 ++index 829464d44..4ac23594b 100644 + --- a/protobuf.bzl + +++ b/protobuf.bzl +-@@ -84,7 +84,9 @@ def _proto_gen_impl(ctx): +- ++@@ -87,6 +87,8 @@ def _proto_gen_impl(ctx): + for dep in ctx.attr.deps: + import_flags += dep.proto.import_flags + deps += dep.proto.deps + + import_flags = depset(import_flags).to_list() + + deps = depset(deps).to_list() +- ++ + if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin: +- return struct( ++ return struct( +diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl +index ead41e83c3..448fbee13f 100755 +--- a/tensorflow/workspace.bzl ++++ b/tensorflow/workspace.bzl +@@ -476,14 +476,14 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): + tf_http_archive( + name = "com_google_protobuf", + patch_file = clean_dep("//third_party/protobuf:protobuf.patch"), +- sha256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b", +- strip_prefix = "protobuf-3.9.2", ++ sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9", ++ strip_prefix = "protobuf-3.11.4", + system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"), + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + }, + urls = [ +- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", +- "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip", ++ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz", ++ "https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz", + ], + )