Compare commits
1 Commits
master
...
dependabot
Author | SHA1 | Date | |
---|---|---|---|
|
a0a3357151 |
3
.bazelrc
3
.bazelrc
|
@ -98,9 +98,6 @@ build:darwin_arm64 --apple_platform_type=macos
|
||||||
build:darwin_arm64 --macos_minimum_os=10.16
|
build:darwin_arm64 --macos_minimum_os=10.16
|
||||||
build:darwin_arm64 --cpu=darwin_arm64
|
build:darwin_arm64 --cpu=darwin_arm64
|
||||||
|
|
||||||
# Turn off maximum stdout size
|
|
||||||
build --experimental_ui_max_stdouterr_bytes=-1
|
|
||||||
|
|
||||||
# This bazelrc file is meant to be written by a setup script.
|
# This bazelrc file is meant to be written by a setup script.
|
||||||
try-import %workspace%/.configure.bazelrc
|
try-import %workspace%/.configure.bazelrc
|
||||||
|
|
||||||
|
|
|
@ -40,16 +40,18 @@ body:
|
||||||
label: Programming Language and version (e.g. C++, Python, Java)
|
label: Programming Language and version (e.g. C++, Python, Java)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
|
@ -41,16 +41,18 @@ body:
|
||||||
label: Task name (e.g. Image classification, Gesture recognition etc.)
|
label: Task name (e.g. Image classification, Gesture recognition etc.)
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
|
@ -31,16 +31,18 @@ body:
|
||||||
label: URL that shows the problem
|
label: URL that shows the problem
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
|
@ -28,33 +28,37 @@ body:
|
||||||
- 'No'
|
- 'No'
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: behaviour
|
id: behaviour
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the feature and the current behaviour/state
|
label: Describe the feature and the current behaviour/state
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: api_change
|
id: api_change
|
||||||
attributes:
|
attributes:
|
||||||
label: Will this change the current API? How?
|
label: Will this change the current API? How?
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: benifit
|
id: benifit
|
||||||
attributes:
|
attributes:
|
||||||
label: Who will benefit with this feature?
|
label: Who will benefit with this feature?
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: use_case
|
id: use_case
|
||||||
attributes:
|
attributes:
|
||||||
label: Please specify the use cases for this feature
|
label: Please specify the use cases for this feature
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: info_other
|
id: info_other
|
||||||
attributes:
|
attributes:
|
||||||
label: Any Other info
|
label: Any Other info
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
|
|
|
@ -87,13 +87,14 @@ body:
|
||||||
placeholder:
|
placeholder:
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: what-happened
|
id: what-happened
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the problem
|
label: Describe the problem
|
||||||
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
|
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
|
||||||
placeholder: Tell us what you see!
|
placeholder: Tell us what you see!
|
||||||
value: "A bug happened!"
|
value: "A bug happened!"
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
|
@ -80,16 +80,18 @@ body:
|
||||||
label: Xcode & Tulsi version (if issue is related to building for iOS)
|
label: Xcode & Tulsi version (if issue is related to building for iOS)
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
- type: textarea
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: true
|
required: true
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
|
@ -48,16 +48,18 @@ body:
|
||||||
placeholder: e.g. C++, Python, Java
|
placeholder: e.g. C++, Python, Java
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: current_model
|
id: current_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the actual behavior
|
label: Describe the actual behavior
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: input
|
- type: textarea
|
||||||
id: expected_model
|
id: expected_model
|
||||||
attributes:
|
attributes:
|
||||||
label: Describe the expected behaviour
|
label: Describe the expected behaviour
|
||||||
|
render: shell
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
- type: textarea
|
- type: textarea
|
||||||
|
|
4
.github/workflows/stale.yaml
vendored
4
.github/workflows/stale.yaml
vendored
|
@ -39,9 +39,7 @@ jobs:
|
||||||
# Limit the No. of API calls in one run default value is 30.
|
# Limit the No. of API calls in one run default value is 30.
|
||||||
operations-per-run: 500
|
operations-per-run: 500
|
||||||
# Prevent to remove stale label when PRs or issues are updated.
|
# Prevent to remove stale label when PRs or issues are updated.
|
||||||
remove-stale-when-updated: true
|
remove-stale-when-updated: false
|
||||||
# List of labels to remove when issues/PRs unstale.
|
|
||||||
labels-to-remove-when-unstale: 'stat:awaiting response'
|
|
||||||
# comment on issue if not active for more then 7 days.
|
# comment on issue if not active for more then 7 days.
|
||||||
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
|
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
|
||||||
# comment on PR if not active for more then 14 days.
|
# comment on PR if not active for more then 14 days.
|
||||||
|
|
50
WORKSPACE
50
WORKSPACE
|
@ -73,9 +73,12 @@ http_archive(
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "zlib",
|
name = "zlib",
|
||||||
build_file = "@//third_party:zlib.BUILD",
|
build_file = "@//third_party:zlib.BUILD",
|
||||||
sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
|
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
|
||||||
strip_prefix = "zlib-1.2.13",
|
strip_prefix = "zlib-1.2.11",
|
||||||
url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
|
urls = [
|
||||||
|
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
|
||||||
|
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
|
||||||
|
],
|
||||||
patches = [
|
patches = [
|
||||||
"@//third_party:zlib.diff",
|
"@//third_party:zlib.diff",
|
||||||
],
|
],
|
||||||
|
@ -176,25 +179,6 @@ http_archive(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
# 2023-06-05
|
|
||||||
# This version of Glog is required for Windows support, but currently causes
|
|
||||||
# crashes on some Android devices.
|
|
||||||
http_archive(
|
|
||||||
name = "com_github_glog_glog_windows",
|
|
||||||
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
|
|
||||||
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
|
|
||||||
urls = [
|
|
||||||
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
|
|
||||||
],
|
|
||||||
patches = [
|
|
||||||
"@//third_party:com_github_glog_glog.diff",
|
|
||||||
"@//third_party:com_github_glog_glog_windows_patch.diff",
|
|
||||||
],
|
|
||||||
patch_args = [
|
|
||||||
"-p1",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# easyexif
|
# easyexif
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "easyexif",
|
name = "easyexif",
|
||||||
|
@ -244,14 +228,16 @@ http_archive(
|
||||||
# sentencepiece
|
# sentencepiece
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "com_google_sentencepiece",
|
name = "com_google_sentencepiece",
|
||||||
strip_prefix = "sentencepiece-0.1.96",
|
strip_prefix = "sentencepiece-1.0.0",
|
||||||
sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
|
sha256 = "c05901f30a1d0ed64cbcf40eba08e48894e1b0e985777217b7c9036cac631346",
|
||||||
urls = [
|
urls = [
|
||||||
"https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip"
|
"https://github.com/google/sentencepiece/archive/1.0.0.zip",
|
||||||
|
],
|
||||||
|
patches = [
|
||||||
|
"@//third_party:com_google_sentencepiece_no_gflag_no_gtest.diff",
|
||||||
],
|
],
|
||||||
build_file = "@//third_party:sentencepiece.BUILD",
|
|
||||||
patches = ["@//third_party:com_google_sentencepiece.diff"],
|
|
||||||
patch_args = ["-p1"],
|
patch_args = ["-p1"],
|
||||||
|
repo_mapping = {"@com_google_glog" : "@com_github_glog_glog_no_gflags"},
|
||||||
)
|
)
|
||||||
|
|
||||||
http_archive(
|
http_archive(
|
||||||
|
@ -499,10 +485,10 @@ http_archive(
|
||||||
)
|
)
|
||||||
|
|
||||||
# TensorFlow repo should always go after the other external dependencies.
|
# TensorFlow repo should always go after the other external dependencies.
|
||||||
# TF on 2023-07-26.
|
# TF on 2023-06-13.
|
||||||
_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd"
|
_TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
|
||||||
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
|
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
|
||||||
_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4"
|
_TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
|
||||||
http_archive(
|
http_archive(
|
||||||
name = "org_tensorflow",
|
name = "org_tensorflow",
|
||||||
urls = [
|
urls = [
|
||||||
|
@ -510,12 +496,8 @@ http_archive(
|
||||||
],
|
],
|
||||||
patches = [
|
patches = [
|
||||||
"@//third_party:org_tensorflow_compatibility_fixes.diff",
|
"@//third_party:org_tensorflow_compatibility_fixes.diff",
|
||||||
"@//third_party:org_tensorflow_system_python.diff",
|
|
||||||
# Diff is generated with a script, don't update it manually.
|
# Diff is generated with a script, don't update it manually.
|
||||||
"@//third_party:org_tensorflow_custom_ops.diff",
|
"@//third_party:org_tensorflow_custom_ops.diff",
|
||||||
# Works around Bazel issue with objc_library.
|
|
||||||
# See https://github.com/bazelbuild/bazel/issues/19912
|
|
||||||
"@//third_party:org_tensorflow_objc_build_fixes.diff",
|
|
||||||
],
|
],
|
||||||
patch_args = [
|
patch_args = [
|
||||||
"-p1",
|
"-p1",
|
||||||
|
|
|
@ -1,342 +0,0 @@
|
||||||
// !$*UTF8*$!
|
|
||||||
{
|
|
||||||
archiveVersion = 1;
|
|
||||||
classes = {
|
|
||||||
};
|
|
||||||
objectVersion = 56;
|
|
||||||
objects = {
|
|
||||||
|
|
||||||
/* Begin PBXBuildFile section */
|
|
||||||
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; };
|
|
||||||
/* End PBXBuildFile section */
|
|
||||||
|
|
||||||
/* Begin PBXFileReference section */
|
|
||||||
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; };
|
|
||||||
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = "<group>"; };
|
|
||||||
/* End PBXFileReference section */
|
|
||||||
|
|
||||||
/* Begin PBXFrameworksBuildPhase section */
|
|
||||||
8566B5562ABABF9A00AAB22A /* Frameworks */ = {
|
|
||||||
isa = PBXFrameworksBuildPhase;
|
|
||||||
buildActionMask = 2147483647;
|
|
||||||
files = (
|
|
||||||
);
|
|
||||||
runOnlyForDeploymentPostprocessing = 0;
|
|
||||||
};
|
|
||||||
/* End PBXFrameworksBuildPhase section */
|
|
||||||
|
|
||||||
/* Begin PBXGroup section */
|
|
||||||
8566B54F2ABABF9A00AAB22A = {
|
|
||||||
isa = PBXGroup;
|
|
||||||
children = (
|
|
||||||
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
|
|
||||||
8566B55A2ABABF9A00AAB22A /* Products */,
|
|
||||||
);
|
|
||||||
sourceTree = "<group>";
|
|
||||||
};
|
|
||||||
8566B55A2ABABF9A00AAB22A /* Products */ = {
|
|
||||||
isa = PBXGroup;
|
|
||||||
children = (
|
|
||||||
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */,
|
|
||||||
);
|
|
||||||
name = Products;
|
|
||||||
sourceTree = "<group>";
|
|
||||||
};
|
|
||||||
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
|
|
||||||
isa = PBXGroup;
|
|
||||||
children = (
|
|
||||||
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */,
|
|
||||||
);
|
|
||||||
path = MediaPipeTasksDocGen;
|
|
||||||
sourceTree = "<group>";
|
|
||||||
};
|
|
||||||
/* End PBXGroup section */
|
|
||||||
|
|
||||||
/* Begin PBXHeadersBuildPhase section */
|
|
||||||
8566B5542ABABF9A00AAB22A /* Headers */ = {
|
|
||||||
isa = PBXHeadersBuildPhase;
|
|
||||||
buildActionMask = 2147483647;
|
|
||||||
files = (
|
|
||||||
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */,
|
|
||||||
);
|
|
||||||
runOnlyForDeploymentPostprocessing = 0;
|
|
||||||
};
|
|
||||||
/* End PBXHeadersBuildPhase section */
|
|
||||||
|
|
||||||
/* Begin PBXNativeTarget section */
|
|
||||||
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
|
|
||||||
isa = PBXNativeTarget;
|
|
||||||
buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */;
|
|
||||||
buildPhases = (
|
|
||||||
8566B5542ABABF9A00AAB22A /* Headers */,
|
|
||||||
8566B5552ABABF9A00AAB22A /* Sources */,
|
|
||||||
8566B5562ABABF9A00AAB22A /* Frameworks */,
|
|
||||||
8566B5572ABABF9A00AAB22A /* Resources */,
|
|
||||||
);
|
|
||||||
buildRules = (
|
|
||||||
);
|
|
||||||
dependencies = (
|
|
||||||
);
|
|
||||||
name = MediaPipeTasksDocGen;
|
|
||||||
productName = MediaPipeTasksDocGen;
|
|
||||||
productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */;
|
|
||||||
productType = "com.apple.product-type.framework";
|
|
||||||
};
|
|
||||||
/* End PBXNativeTarget section */
|
|
||||||
|
|
||||||
/* Begin PBXProject section */
|
|
||||||
8566B5502ABABF9A00AAB22A /* Project object */ = {
|
|
||||||
isa = PBXProject;
|
|
||||||
attributes = {
|
|
||||||
BuildIndependentTargetsInParallel = 1;
|
|
||||||
LastUpgradeCheck = 1430;
|
|
||||||
TargetAttributes = {
|
|
||||||
8566B5582ABABF9A00AAB22A = {
|
|
||||||
CreatedOnToolsVersion = 14.3.1;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */;
|
|
||||||
compatibilityVersion = "Xcode 14.0";
|
|
||||||
developmentRegion = en;
|
|
||||||
hasScannedForEncodings = 0;
|
|
||||||
knownRegions = (
|
|
||||||
en,
|
|
||||||
Base,
|
|
||||||
);
|
|
||||||
mainGroup = 8566B54F2ABABF9A00AAB22A;
|
|
||||||
productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */;
|
|
||||||
projectDirPath = "";
|
|
||||||
projectRoot = "";
|
|
||||||
targets = (
|
|
||||||
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
|
|
||||||
);
|
|
||||||
};
|
|
||||||
/* End PBXProject section */
|
|
||||||
|
|
||||||
/* Begin PBXResourcesBuildPhase section */
|
|
||||||
8566B5572ABABF9A00AAB22A /* Resources */ = {
|
|
||||||
isa = PBXResourcesBuildPhase;
|
|
||||||
buildActionMask = 2147483647;
|
|
||||||
files = (
|
|
||||||
);
|
|
||||||
runOnlyForDeploymentPostprocessing = 0;
|
|
||||||
};
|
|
||||||
/* End PBXResourcesBuildPhase section */
|
|
||||||
|
|
||||||
/* Begin PBXSourcesBuildPhase section */
|
|
||||||
8566B5552ABABF9A00AAB22A /* Sources */ = {
|
|
||||||
isa = PBXSourcesBuildPhase;
|
|
||||||
buildActionMask = 2147483647;
|
|
||||||
files = (
|
|
||||||
);
|
|
||||||
runOnlyForDeploymentPostprocessing = 0;
|
|
||||||
};
|
|
||||||
/* End PBXSourcesBuildPhase section */
|
|
||||||
|
|
||||||
/* Begin XCBuildConfiguration section */
|
|
||||||
8566B55E2ABABF9A00AAB22A /* Debug */ = {
|
|
||||||
isa = XCBuildConfiguration;
|
|
||||||
buildSettings = {
|
|
||||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
|
||||||
CLANG_ANALYZER_NONNULL = YES;
|
|
||||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
|
||||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
|
||||||
CLANG_ENABLE_MODULES = YES;
|
|
||||||
CLANG_ENABLE_OBJC_ARC = YES;
|
|
||||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
|
||||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
|
||||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_COMMA = YES;
|
|
||||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
|
||||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
|
||||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
|
||||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
|
||||||
CLANG_WARN_EMPTY_BODY = YES;
|
|
||||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
|
||||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
|
||||||
CLANG_WARN_INT_CONVERSION = YES;
|
|
||||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
|
||||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
|
||||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
|
||||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
|
||||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
|
||||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
|
||||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
|
||||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
|
||||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
|
||||||
COPY_PHASE_STRIP = NO;
|
|
||||||
CURRENT_PROJECT_VERSION = 1;
|
|
||||||
DEBUG_INFORMATION_FORMAT = dwarf;
|
|
||||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
|
||||||
ENABLE_TESTABILITY = YES;
|
|
||||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
|
||||||
GCC_DYNAMIC_NO_PIC = NO;
|
|
||||||
GCC_NO_COMMON_BLOCKS = YES;
|
|
||||||
GCC_OPTIMIZATION_LEVEL = 0;
|
|
||||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
|
||||||
"DEBUG=1",
|
|
||||||
"$(inherited)",
|
|
||||||
);
|
|
||||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
|
||||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
|
||||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
|
||||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
|
||||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
|
||||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
|
||||||
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
|
||||||
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
|
||||||
MTL_FAST_MATH = YES;
|
|
||||||
ONLY_ACTIVE_ARCH = YES;
|
|
||||||
SDKROOT = iphoneos;
|
|
||||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
|
||||||
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
|
||||||
VERSIONING_SYSTEM = "apple-generic";
|
|
||||||
VERSION_INFO_PREFIX = "";
|
|
||||||
};
|
|
||||||
name = Debug;
|
|
||||||
};
|
|
||||||
8566B55F2ABABF9A00AAB22A /* Release */ = {
|
|
||||||
isa = XCBuildConfiguration;
|
|
||||||
buildSettings = {
|
|
||||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
|
||||||
CLANG_ANALYZER_NONNULL = YES;
|
|
||||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
|
||||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
|
||||||
CLANG_ENABLE_MODULES = YES;
|
|
||||||
CLANG_ENABLE_OBJC_ARC = YES;
|
|
||||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
|
||||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
|
||||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_COMMA = YES;
|
|
||||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
|
||||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
|
||||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
|
||||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
|
||||||
CLANG_WARN_EMPTY_BODY = YES;
|
|
||||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
|
||||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
|
||||||
CLANG_WARN_INT_CONVERSION = YES;
|
|
||||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
|
||||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
|
||||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
|
||||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
|
||||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
|
||||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
|
||||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
|
||||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
|
||||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
|
||||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
|
||||||
COPY_PHASE_STRIP = NO;
|
|
||||||
CURRENT_PROJECT_VERSION = 1;
|
|
||||||
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
|
||||||
ENABLE_NS_ASSERTIONS = NO;
|
|
||||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
|
||||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
|
||||||
GCC_NO_COMMON_BLOCKS = YES;
|
|
||||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
|
||||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
|
||||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
|
||||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
|
||||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
|
||||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
|
||||||
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
|
||||||
MTL_ENABLE_DEBUG_INFO = NO;
|
|
||||||
MTL_FAST_MATH = YES;
|
|
||||||
SDKROOT = iphoneos;
|
|
||||||
SWIFT_COMPILATION_MODE = wholemodule;
|
|
||||||
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
|
||||||
VALIDATE_PRODUCT = YES;
|
|
||||||
VERSIONING_SYSTEM = "apple-generic";
|
|
||||||
VERSION_INFO_PREFIX = "";
|
|
||||||
};
|
|
||||||
name = Release;
|
|
||||||
};
|
|
||||||
8566B5612ABABF9A00AAB22A /* Debug */ = {
|
|
||||||
isa = XCBuildConfiguration;
|
|
||||||
buildSettings = {
|
|
||||||
CODE_SIGN_STYLE = Automatic;
|
|
||||||
CURRENT_PROJECT_VERSION = 1;
|
|
||||||
DEFINES_MODULE = YES;
|
|
||||||
DYLIB_COMPATIBILITY_VERSION = 1;
|
|
||||||
DYLIB_CURRENT_VERSION = 1;
|
|
||||||
DYLIB_INSTALL_NAME_BASE = "@rpath";
|
|
||||||
ENABLE_MODULE_VERIFIER = YES;
|
|
||||||
GENERATE_INFOPLIST_FILE = YES;
|
|
||||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
|
||||||
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
|
|
||||||
LD_RUNPATH_SEARCH_PATHS = (
|
|
||||||
"$(inherited)",
|
|
||||||
"@executable_path/Frameworks",
|
|
||||||
"@loader_path/Frameworks",
|
|
||||||
);
|
|
||||||
MARKETING_VERSION = 1.0;
|
|
||||||
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
|
|
||||||
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
|
|
||||||
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
|
|
||||||
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
|
|
||||||
SKIP_INSTALL = YES;
|
|
||||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
|
||||||
SWIFT_VERSION = 5.0;
|
|
||||||
TARGETED_DEVICE_FAMILY = "1,2";
|
|
||||||
};
|
|
||||||
name = Debug;
|
|
||||||
};
|
|
||||||
8566B5622ABABF9A00AAB22A /* Release */ = {
|
|
||||||
isa = XCBuildConfiguration;
|
|
||||||
buildSettings = {
|
|
||||||
CODE_SIGN_STYLE = Automatic;
|
|
||||||
CURRENT_PROJECT_VERSION = 1;
|
|
||||||
DEFINES_MODULE = YES;
|
|
||||||
DYLIB_COMPATIBILITY_VERSION = 1;
|
|
||||||
DYLIB_CURRENT_VERSION = 1;
|
|
||||||
DYLIB_INSTALL_NAME_BASE = "@rpath";
|
|
||||||
ENABLE_MODULE_VERIFIER = YES;
|
|
||||||
GENERATE_INFOPLIST_FILE = YES;
|
|
||||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
|
||||||
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
|
|
||||||
LD_RUNPATH_SEARCH_PATHS = (
|
|
||||||
"$(inherited)",
|
|
||||||
"@executable_path/Frameworks",
|
|
||||||
"@loader_path/Frameworks",
|
|
||||||
);
|
|
||||||
MARKETING_VERSION = 1.0;
|
|
||||||
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
|
|
||||||
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
|
|
||||||
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
|
|
||||||
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
|
|
||||||
SKIP_INSTALL = YES;
|
|
||||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
|
||||||
SWIFT_VERSION = 5.0;
|
|
||||||
TARGETED_DEVICE_FAMILY = "1,2";
|
|
||||||
};
|
|
||||||
name = Release;
|
|
||||||
};
|
|
||||||
/* End XCBuildConfiguration section */
|
|
||||||
|
|
||||||
/* Begin XCConfigurationList section */
|
|
||||||
8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = {
|
|
||||||
isa = XCConfigurationList;
|
|
||||||
buildConfigurations = (
|
|
||||||
8566B55E2ABABF9A00AAB22A /* Debug */,
|
|
||||||
8566B55F2ABABF9A00AAB22A /* Release */,
|
|
||||||
);
|
|
||||||
defaultConfigurationIsVisible = 0;
|
|
||||||
defaultConfigurationName = Release;
|
|
||||||
};
|
|
||||||
8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = {
|
|
||||||
isa = XCConfigurationList;
|
|
||||||
buildConfigurations = (
|
|
||||||
8566B5612ABABF9A00AAB22A /* Debug */,
|
|
||||||
8566B5622ABABF9A00AAB22A /* Release */,
|
|
||||||
);
|
|
||||||
defaultConfigurationIsVisible = 0;
|
|
||||||
defaultConfigurationName = Release;
|
|
||||||
};
|
|
||||||
/* End XCConfigurationList section */
|
|
||||||
};
|
|
||||||
rootObject = 8566B5502ABABF9A00AAB22A /* Project object */;
|
|
||||||
}
|
|
|
@ -1,7 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<Workspace
|
|
||||||
version = "1.0">
|
|
||||||
<FileRef
|
|
||||||
location = "self:">
|
|
||||||
</FileRef>
|
|
||||||
</Workspace>
|
|
|
@ -1,8 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
||||||
<plist version="1.0">
|
|
||||||
<dict>
|
|
||||||
<key>IDEDidComputeMac32BitWarning</key>
|
|
||||||
<true/>
|
|
||||||
</dict>
|
|
||||||
</plist>
|
|
Binary file not shown.
|
@ -1,14 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
||||||
<plist version="1.0">
|
|
||||||
<dict>
|
|
||||||
<key>SchemeUserState</key>
|
|
||||||
<dict>
|
|
||||||
<key>MediaPipeTasksDocGen.xcscheme_^#shared#^_</key>
|
|
||||||
<dict>
|
|
||||||
<key>orderHint</key>
|
|
||||||
<integer>0</integer>
|
|
||||||
</dict>
|
|
||||||
</dict>
|
|
||||||
</dict>
|
|
||||||
</plist>
|
|
|
@ -1,17 +0,0 @@
|
||||||
//
|
|
||||||
// MediaPipeTasksDocGen.h
|
|
||||||
// MediaPipeTasksDocGen
|
|
||||||
//
|
|
||||||
// Created by Mark McDonald on 20/9/2023.
|
|
||||||
//
|
|
||||||
|
|
||||||
#import <Foundation/Foundation.h>
|
|
||||||
|
|
||||||
//! Project version number for MediaPipeTasksDocGen.
|
|
||||||
FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber;
|
|
||||||
|
|
||||||
//! Project version string for MediaPipeTasksDocGen.
|
|
||||||
FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[];
|
|
||||||
|
|
||||||
// In this header, you should import all the public headers of your framework using statements like
|
|
||||||
// #import <MediaPipeTasksDocGen/PublicHeader.h>
|
|
|
@ -1,11 +0,0 @@
|
||||||
# Uncomment the next line to define a global platform for your project
|
|
||||||
platform :ios, '15.0'
|
|
||||||
|
|
||||||
target 'MediaPipeTasksDocGen' do
|
|
||||||
# Comment the next line if you don't want to use dynamic frameworks
|
|
||||||
use_frameworks!
|
|
||||||
|
|
||||||
# Pods for MediaPipeTasksDocGen
|
|
||||||
pod 'MediaPipeTasksText'
|
|
||||||
pod 'MediaPipeTasksVision'
|
|
||||||
end
|
|
|
@ -1,9 +0,0 @@
|
||||||
# MediaPipeTasksDocGen
|
|
||||||
|
|
||||||
This empty project is used to generate reference documentation for the
|
|
||||||
ObjectiveC and Swift libraries.
|
|
||||||
|
|
||||||
Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published
|
|
||||||
to [the developer site](https://developers.google.com/mediapipe/solutions/).
|
|
||||||
|
|
||||||
To bump the API version used, edit [`Podfile`](./Podfile).
|
|
|
@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||||
3. The [`hello world`] example uses a simple MediaPipe graph in the
|
3. The [`hello world`] example uses a simple MediaPipe graph in the
|
||||||
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
|
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
|
||||||
|
|
||||||
```c++
|
```C++
|
||||||
absl::Status PrintHelloWorld() {
|
absl::Status PrintHelloWorld() {
|
||||||
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
|
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
|
||||||
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
|
||||||
|
@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
|
||||||
```c++
|
```c++
|
||||||
mediapipe::Packet packet;
|
mediapipe::Packet packet;
|
||||||
while (poller.Next(&packet)) {
|
while (poller.Next(&packet)) {
|
||||||
ABSL_LOG(INFO) << packet.Get<string>();
|
LOG(INFO) << packet.Get<string>();
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
|
||||||
rules:
|
rules:
|
||||||
|
|
||||||
```
|
```
|
||||||
MIN_IOS_VERSION = "12.0"
|
MIN_IOS_VERSION = "11.0"
|
||||||
|
|
||||||
load(
|
load(
|
||||||
"@build_bazel_rules_apple//apple:ios.bzl",
|
"@build_bazel_rules_apple//apple:ios.bzl",
|
||||||
|
|
254
mediapipe/BUILD
254
mediapipe/BUILD
|
@ -14,155 +14,81 @@
|
||||||
|
|
||||||
licenses(["notice"]) # Apache 2.0
|
licenses(["notice"]) # Apache 2.0
|
||||||
|
|
||||||
load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform")
|
# Note: yes, these need to use "//external:android/crosstool", not
|
||||||
|
# @androidndk//:default_crosstool.
|
||||||
|
|
||||||
# Generic Android
|
|
||||||
config_setting(
|
config_setting(
|
||||||
name = "android",
|
name = "android",
|
||||||
constraint_values = [
|
values = {"crosstool_top": "//external:android/crosstool"},
|
||||||
"@platforms//os:android",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Android x86 32-bit.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "android_x86",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:android",
|
|
||||||
"@platforms//cpu:x86_32",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Android x86 64-bit.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "android_x86_64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:android",
|
|
||||||
"@platforms//cpu:x86_64",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Android ARMv7.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "android_arm",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:android",
|
|
||||||
"@platforms//cpu:armv7",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Android ARM64.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "android_arm64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:android",
|
|
||||||
"@platforms//cpu:arm64",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Generic MacOS.
|
|
||||||
config_setting(
|
config_setting(
|
||||||
|
name = "android_x86",
|
||||||
|
values = {
|
||||||
|
"crosstool_top": "//external:android/crosstool",
|
||||||
|
"cpu": "x86",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "android_x86_64",
|
||||||
|
values = {
|
||||||
|
"crosstool_top": "//external:android/crosstool",
|
||||||
|
"cpu": "x86_64",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "android_armeabi",
|
||||||
|
values = {
|
||||||
|
"crosstool_top": "//external:android/crosstool",
|
||||||
|
"cpu": "armeabi",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "android_arm",
|
||||||
|
values = {
|
||||||
|
"crosstool_top": "//external:android/crosstool",
|
||||||
|
"cpu": "armeabi-v7a",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "android_arm64",
|
||||||
|
values = {
|
||||||
|
"crosstool_top": "//external:android/crosstool",
|
||||||
|
"cpu": "arm64-v8a",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Note: this cannot just match "apple_platform_type": "macos" because that option
|
||||||
|
# defaults to "macos" even when building on Linux!
|
||||||
|
alias(
|
||||||
name = "macos",
|
name = "macos",
|
||||||
constraint_values = [
|
actual = select({
|
||||||
"@platforms//os:macos",
|
":macos_i386": ":macos_i386",
|
||||||
],
|
":macos_x86_64": ":macos_x86_64",
|
||||||
|
":macos_arm64": ":macos_arm64",
|
||||||
|
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
|
||||||
|
}),
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# MacOS x86 64-bit.
|
# Note: this also matches on crosstool_top so that it does not produce ambiguous
|
||||||
config_setting_and_platform(
|
# selectors when used together with "android".
|
||||||
name = "macos_x86_64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:macos",
|
|
||||||
"@platforms//cpu:x86_64",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# MacOS ARM64.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "macos_arm64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:macos",
|
|
||||||
"@platforms//cpu:arm64",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Generic iOS.
|
|
||||||
config_setting(
|
config_setting(
|
||||||
name = "ios",
|
name = "ios",
|
||||||
constraint_values = [
|
values = {
|
||||||
"@platforms//os:ios",
|
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
|
||||||
],
|
"apple_platform_type": "ios",
|
||||||
visibility = ["//visibility:public"],
|
},
|
||||||
)
|
|
||||||
|
|
||||||
# iOS device ARM32.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_armv7",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:arm",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# iOS device ARM64.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_arm64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:arm64",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# iOS device ARM64E.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_arm64e",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:arm64e",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# iOS simulator x86 32-bit.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_i386",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:x86_32",
|
|
||||||
"@build_bazel_apple_support//constraints:simulator",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# iOS simulator x86 64-bit.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_x86_64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:x86_64",
|
|
||||||
"@build_bazel_apple_support//constraints:simulator",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
)
|
|
||||||
|
|
||||||
# iOS simulator ARM64.
|
|
||||||
config_setting_and_platform(
|
|
||||||
name = "ios_sim_arm64",
|
|
||||||
constraint_values = [
|
|
||||||
"@platforms//os:ios",
|
|
||||||
"@platforms//cpu:arm64",
|
|
||||||
"@build_bazel_apple_support//constraints:simulator",
|
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -176,26 +102,54 @@ alias(
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Windows 64-bit.
|
config_setting(
|
||||||
config_setting_and_platform(
|
name = "macos_i386",
|
||||||
name = "windows",
|
values = {
|
||||||
constraint_values = [
|
"apple_platform_type": "macos",
|
||||||
"@platforms//os:windows",
|
"cpu": "darwin",
|
||||||
"@platforms//cpu:x86_64",
|
},
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Linux 64-bit.
|
config_setting(
|
||||||
config_setting_and_platform(
|
name = "macos_x86_64",
|
||||||
name = "linux",
|
values = {
|
||||||
constraint_values = [
|
"apple_platform_type": "macos",
|
||||||
"@platforms//os:linux",
|
"cpu": "darwin_x86_64",
|
||||||
"@platforms//cpu:x86_64",
|
},
|
||||||
],
|
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "macos_arm64",
|
||||||
|
values = {
|
||||||
|
"apple_platform_type": "macos",
|
||||||
|
"cpu": "darwin_arm64",
|
||||||
|
},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
[
|
||||||
|
config_setting(
|
||||||
|
name = arch,
|
||||||
|
values = {"cpu": arch},
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
for arch in [
|
||||||
|
"ios_i386",
|
||||||
|
"ios_x86_64",
|
||||||
|
"ios_armv7",
|
||||||
|
"ios_arm64",
|
||||||
|
"ios_arm64e",
|
||||||
|
"ios_sim_arm64",
|
||||||
|
]
|
||||||
|
]
|
||||||
|
|
||||||
|
config_setting(
|
||||||
|
name = "windows",
|
||||||
|
values = {"cpu": "x64_windows"},
|
||||||
|
)
|
||||||
|
|
||||||
exports_files(
|
exports_files(
|
||||||
["provisioning_profile.mobileprovision"],
|
["provisioning_profile.mobileprovision"],
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# Placeholder: load py_proto_library
|
|
||||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
|
||||||
|
|
||||||
licenses(["notice"])
|
licenses(["notice"])
|
||||||
|
@ -146,7 +145,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_audio_tools//audio/dsp/mfcc",
|
"@com_google_audio_tools//audio/dsp/mfcc",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
|
@ -165,9 +163,8 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_audio_tools//audio/dsp:resampler",
|
"@com_google_audio_tools//audio/dsp:resampler",
|
||||||
"@com_google_audio_tools//audio/dsp:resampler_q",
|
"@com_google_audio_tools//audio/dsp:resampler_q",
|
||||||
|
@ -188,7 +185,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:core_proto",
|
"//mediapipe/framework/port:core_proto",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -228,7 +224,6 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
|
@ -299,7 +294,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_test_util",
|
"//mediapipe/util:time_series_test_util",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_audio_tools//audio/dsp:number_util",
|
"@com_google_audio_tools//audio/dsp:number_util",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
|
@ -333,7 +327,6 @@ cc_binary(
|
||||||
"//mediapipe/framework:packet",
|
"//mediapipe/framework:packet",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_benchmark//:benchmark",
|
"@com_google_benchmark//:benchmark",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -352,7 +345,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:time_series_test_util",
|
"//mediapipe/util:time_series_test_util",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_audio_tools//audio/dsp:window_functions",
|
"@com_google_audio_tools//audio/dsp:window_functions",
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
|
|
|
@ -23,7 +23,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "absl/strings/string_view.h"
|
#include "absl/strings/string_view.h"
|
||||||
#include "absl/strings/substitute.h"
|
#include "absl/strings/substitute.h"
|
||||||
|
@ -139,7 +138,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
|
||||||
TransformFrame(input_frame, &output_frame);
|
TransformFrame(input_frame, &output_frame);
|
||||||
|
|
||||||
// Copy output from vector<float> to Eigen::Vector.
|
// Copy output from vector<float> to Eigen::Vector.
|
||||||
ABSL_CHECK_EQ(output_frame.size(), num_output_channels_);
|
CHECK_EQ(output_frame.size(), num_output_channels_);
|
||||||
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
|
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
|
||||||
output_frame.size(), 1);
|
output_frame.size(), 1);
|
||||||
output->col(frame) = output_frame_map.cast<float>();
|
output->col(frame) = output_frame_map.cast<float>();
|
||||||
|
|
|
@ -16,8 +16,6 @@
|
||||||
|
|
||||||
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "audio/dsp/resampler_q.h"
|
#include "audio/dsp/resampler_q.h"
|
||||||
|
|
||||||
using audio_dsp::Resampler;
|
using audio_dsp::Resampler;
|
||||||
|
@ -47,9 +45,9 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
|
||||||
if (matrix->cols() == 0) {
|
if (matrix->cols() == 0) {
|
||||||
matrix->resize(matrix->rows(), vec.size());
|
matrix->resize(matrix->rows(), vec.size());
|
||||||
} else {
|
} else {
|
||||||
ABSL_CHECK_EQ(vec.size(), matrix->cols());
|
CHECK_EQ(vec.size(), matrix->cols());
|
||||||
}
|
}
|
||||||
ABSL_CHECK_LT(channel, matrix->rows());
|
CHECK_LT(channel, matrix->rows());
|
||||||
matrix->row(channel) =
|
matrix->row(channel) =
|
||||||
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
|
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
|
||||||
}
|
}
|
||||||
|
@ -79,7 +77,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
|
||||||
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
|
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
|
||||||
resample_options);
|
resample_options);
|
||||||
if (!r) {
|
if (!r) {
|
||||||
ABSL_LOG(ERROR) << "Failed to initialize resampler.";
|
LOG(ERROR) << "Failed to initialize resampler.";
|
||||||
return absl::UnknownError("Failed to initialize resampler.");
|
return absl::UnknownError("Failed to initialize resampler.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,6 +27,7 @@
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
#include "mediapipe/framework/formats/time_series_header.pb.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/util/time_series_util.h"
|
#include "mediapipe/util/time_series_util.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|
|
@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
|
||||||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||||
// the last sample of the packet and it's inferred from the latest input
|
// the last sample of the packet and it's inferred from the latest input
|
||||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||||
// the cumulative timestamping, which is inferred from the initial input
|
// the cumulative timestamping, which is inferred from the intial input
|
||||||
// timestamp and the cumulative number of samples.
|
// timestamp and the cumulative number of samples.
|
||||||
optional bool use_local_timestamp = 8 [default = false];
|
optional bool use_local_timestamp = 8 [default = false];
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "audio/dsp/number_util.h"
|
#include "audio/dsp/number_util.h"
|
||||||
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -883,11 +882,11 @@ void BM_ProcessDC(benchmark::State& state) {
|
||||||
|
|
||||||
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
|
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
|
||||||
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
|
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
|
||||||
ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
|
LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
|
||||||
<< output_matrix.cols();
|
<< output_matrix.cols();
|
||||||
ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
|
LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
|
||||||
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
|
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
|
||||||
<< output_matrix(3, 0);
|
<< output_matrix(3, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
BENCHMARK(BM_ProcessDC);
|
BENCHMARK(BM_ProcessDC);
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
|
@ -60,7 +59,7 @@ class StabilizedLogCalculator : public CalculatorBase {
|
||||||
output_scale_ = stabilized_log_calculator_options.output_scale();
|
output_scale_ = stabilized_log_calculator_options.output_scale();
|
||||||
check_nonnegativity_ =
|
check_nonnegativity_ =
|
||||||
stabilized_log_calculator_options.check_nonnegativity();
|
stabilized_log_calculator_options.check_nonnegativity();
|
||||||
ABSL_CHECK_GE(stabilizer_, 0.0)
|
CHECK_GE(stabilizer_, 0.0)
|
||||||
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
|
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
|
||||||
|
|
||||||
// If the input packets have a header, propagate the header to the output.
|
// If the input packets have a header, propagate the header to the output.
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "audio/dsp/window_functions.h"
|
#include "audio/dsp/window_functions.h"
|
||||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -105,7 +104,7 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
|
||||||
// All numbers are in input samples.
|
// All numbers are in input samples.
|
||||||
const int64_t current_output_frame_start = static_cast<int64_t>(
|
const int64_t current_output_frame_start = static_cast<int64_t>(
|
||||||
round(cumulative_output_frames_ * average_frame_step_samples_));
|
round(cumulative_output_frames_ * average_frame_step_samples_));
|
||||||
ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
|
CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
|
||||||
const int64_t next_output_frame_start = static_cast<int64_t>(
|
const int64_t next_output_frame_start = static_cast<int64_t>(
|
||||||
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
|
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
|
||||||
return next_output_frame_start - current_output_frame_start;
|
return next_output_frame_start - current_output_frame_start;
|
||||||
|
|
|
@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
|
||||||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||||
// the last sample of the packet and it's inferred from the latest input
|
// the last sample of the packet and it's inferred from the latest input
|
||||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||||
// the cumulative timestamping, which is inferred from the initial input
|
// the cumulative timestamping, which is inferred from the intial input
|
||||||
// timestamp and the cumulative number of samples.
|
// timestamp and the cumulative number of samples.
|
||||||
optional bool use_local_timestamp = 6 [default = false];
|
optional bool use_local_timestamp = 6 [default = false];
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
#include <random>
|
#include <random>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "benchmark/benchmark.h"
|
#include "benchmark/benchmark.h"
|
||||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -71,7 +70,7 @@ void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
|
||||||
}
|
}
|
||||||
// Initialize graph.
|
// Initialize graph.
|
||||||
mediapipe::CalculatorGraph graph;
|
mediapipe::CalculatorGraph graph;
|
||||||
ABSL_CHECK_OK(graph.Initialize(config));
|
CHECK_OK(graph.Initialize(config));
|
||||||
// Prepare input header.
|
// Prepare input header.
|
||||||
auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
|
auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
|
||||||
header->set_sample_rate(kSampleRate);
|
header->set_sample_rate(kSampleRate);
|
||||||
|
@ -79,13 +78,13 @@ void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
|
||||||
|
|
||||||
state.ResumeTiming(); // Resume benchmark timing.
|
state.ResumeTiming(); // Resume benchmark timing.
|
||||||
|
|
||||||
ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
|
CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
|
||||||
for (auto& packet : input_packets) {
|
for (auto& packet : input_packets) {
|
||||||
ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet));
|
CHECK_OK(graph.AddPacketToInputStream("input", packet));
|
||||||
}
|
}
|
||||||
ABSL_CHECK(!graph.HasError());
|
CHECK(!graph.HasError());
|
||||||
ABSL_CHECK_OK(graph.CloseAllInputStreams());
|
CHECK_OK(graph.CloseAllInputStreams());
|
||||||
ABSL_CHECK_OK(graph.WaitUntilIdle());
|
CHECK_OK(graph.WaitUntilIdle());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
BENCHMARK(BM_TimeSeriesFramerCalculator);
|
BENCHMARK(BM_TimeSeriesFramerCalculator);
|
||||||
|
|
|
@ -19,7 +19,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "audio/dsp/window_functions.h"
|
#include "audio/dsp/window_functions.h"
|
||||||
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -187,12 +186,11 @@ class TimeSeriesFramerCalculatorTest
|
||||||
const int num_unique_output_samples =
|
const int num_unique_output_samples =
|
||||||
round((output().packets.size() - 1) * frame_step_samples) +
|
round((output().packets.size() - 1) * frame_step_samples) +
|
||||||
frame_duration_samples;
|
frame_duration_samples;
|
||||||
ABSL_LOG(INFO) << "packets.size()=" << output().packets.size()
|
LOG(INFO) << "packets.size()=" << output().packets.size()
|
||||||
<< " frame_duration_samples=" << frame_duration_samples
|
<< " frame_duration_samples=" << frame_duration_samples
|
||||||
<< " frame_step_samples=" << frame_step_samples
|
<< " frame_step_samples=" << frame_step_samples
|
||||||
<< " num_input_samples_=" << num_input_samples_
|
<< " num_input_samples_=" << num_input_samples_
|
||||||
<< " num_unique_output_samples="
|
<< " num_unique_output_samples=" << num_unique_output_samples;
|
||||||
<< num_unique_output_samples;
|
|
||||||
const int num_padding_samples =
|
const int num_padding_samples =
|
||||||
num_unique_output_samples - num_input_samples_;
|
num_unique_output_samples - num_input_samples_;
|
||||||
if (options_.pad_final_packet()) {
|
if (options_.pad_final_packet()) {
|
||||||
|
|
|
@ -21,10 +21,10 @@ licenses(["notice"])
|
||||||
package(default_visibility = ["//visibility:public"])
|
package(default_visibility = ["//visibility:public"])
|
||||||
|
|
||||||
selects.config_setting_group(
|
selects.config_setting_group(
|
||||||
name = "apple_or_disable_gpu",
|
name = "ios_or_disable_gpu",
|
||||||
match_any = [
|
match_any = [
|
||||||
"//mediapipe/gpu:disable_gpu",
|
"//mediapipe/gpu:disable_gpu",
|
||||||
"//mediapipe:apple",
|
"//mediapipe:ios",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -299,7 +299,7 @@ cc_library(
|
||||||
"//mediapipe/util:render_data_cc_proto",
|
"//mediapipe/util:render_data_cc_proto",
|
||||||
"@org_tensorflow//tensorflow/lite:framework",
|
"@org_tensorflow//tensorflow/lite:framework",
|
||||||
] + select({
|
] + select({
|
||||||
":apple_or_disable_gpu": [],
|
":ios_or_disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||||
],
|
],
|
||||||
|
@ -325,7 +325,6 @@ cc_library(
|
||||||
":concatenate_vector_calculator_cc_proto",
|
":concatenate_vector_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/formats:body_rig_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:classification_cc_proto",
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
|
@ -382,6 +381,17 @@ cc_library(
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "clip_detection_vector_size_calculator",
|
||||||
|
srcs = ["clip_detection_vector_size_calculator.cc"],
|
||||||
|
deps = [
|
||||||
|
":clip_vector_size_calculator",
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
|
],
|
||||||
|
alwayslink = 1,
|
||||||
|
)
|
||||||
|
|
||||||
cc_test(
|
cc_test(
|
||||||
name = "clip_vector_size_calculator_test",
|
name = "clip_vector_size_calculator_test",
|
||||||
srcs = ["clip_vector_size_calculator_test.cc"],
|
srcs = ["clip_vector_size_calculator_test.cc"],
|
||||||
|
@ -583,7 +593,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/tool:options_util",
|
"//mediapipe/framework/tool:options_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -599,7 +608,6 @@ cc_test(
|
||||||
"//mediapipe/framework/formats:video_stream_header",
|
"//mediapipe/framework/formats:video_stream_header",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -632,7 +640,6 @@ cc_library(
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -727,7 +734,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -743,7 +749,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/tool:options_util",
|
"//mediapipe/framework/tool:options_util",
|
||||||
"//mediapipe/util:packet_test_util",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
|
@ -782,11 +787,10 @@ cc_library(
|
||||||
"//mediapipe/framework/deps:random",
|
"//mediapipe/framework/deps:random",
|
||||||
"//mediapipe/framework/formats:video_stream_header",
|
"//mediapipe/framework/formats:video_stream_header",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/tool:options_util",
|
"//mediapipe/framework/tool:options_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
|
@ -842,7 +846,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/tool:validate_type",
|
"//mediapipe/framework/tool:validate_type",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@eigen_archive//:eigen3",
|
"@eigen_archive//:eigen3",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -915,7 +918,7 @@ cc_library(
|
||||||
"@org_tensorflow//tensorflow/lite:framework",
|
"@org_tensorflow//tensorflow/lite:framework",
|
||||||
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
|
||||||
] + select({
|
] + select({
|
||||||
":apple_or_disable_gpu": [],
|
":ios_or_disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
|
||||||
],
|
],
|
||||||
|
@ -947,7 +950,6 @@ cc_library(
|
||||||
deps = [
|
deps = [
|
||||||
":split_vector_calculator_cc_proto",
|
":split_vector_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/formats:body_rig_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
|
@ -1031,7 +1033,6 @@ cc_library(
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -1070,7 +1071,6 @@ cc_test(
|
||||||
"//mediapipe/framework:calculator_runner",
|
"//mediapipe/framework:calculator_runner",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -1117,7 +1117,6 @@ cc_library(
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -1393,26 +1392,3 @@ cc_test(
|
||||||
"@com_google_absl//absl/types:optional",
|
"@com_google_absl//absl/types:optional",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "value_or_default_calculator",
|
|
||||||
srcs = ["value_or_default_calculator.cc"],
|
|
||||||
visibility = ["//visibility:public"],
|
|
||||||
deps = [
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
],
|
|
||||||
alwayslink = True,
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "value_or_default_calculator_test",
|
|
||||||
srcs = ["value_or_default_calculator_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":value_or_default_calculator",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework:calculator_runner",
|
|
||||||
"//mediapipe/framework:packet",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
|
@ -13,7 +13,6 @@
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
|
@ -164,75 +163,6 @@ TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
|
||||||
PacketOfIntsEq(input_timestamp2, std::vector<int>{3, 4})));
|
PacketOfIntsEq(input_timestamp2, std::vector<int>{3, 4})));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(BeginEndLoopCalculatorPossibleDataRaceTest,
|
|
||||||
EndLoopForIntegersDoesNotRace) {
|
|
||||||
auto graph_config = ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
R"pb(
|
|
||||||
num_threads: 4
|
|
||||||
input_stream: "ints"
|
|
||||||
node {
|
|
||||||
calculator: "BeginLoopIntegerCalculator"
|
|
||||||
input_stream: "ITERABLE:ints"
|
|
||||||
output_stream: "ITEM:int"
|
|
||||||
output_stream: "BATCH_END:timestamp"
|
|
||||||
}
|
|
||||||
node {
|
|
||||||
calculator: "IncrementCalculator"
|
|
||||||
input_stream: "int"
|
|
||||||
output_stream: "int_plus_one"
|
|
||||||
}
|
|
||||||
# BEGIN: Data race possibility
|
|
||||||
# EndLoop###Calculator and another calculator using the same input
|
|
||||||
# may introduce race due to EndLoop###Calculator possibly consuming
|
|
||||||
# packet.
|
|
||||||
node {
|
|
||||||
calculator: "EndLoopIntegersCalculator"
|
|
||||||
input_stream: "ITEM:int_plus_one"
|
|
||||||
input_stream: "BATCH_END:timestamp"
|
|
||||||
output_stream: "ITERABLE:ints_plus_one"
|
|
||||||
}
|
|
||||||
node {
|
|
||||||
calculator: "IncrementCalculator"
|
|
||||||
input_stream: "int_plus_one"
|
|
||||||
output_stream: "int_plus_two"
|
|
||||||
}
|
|
||||||
# END: Data race possibility
|
|
||||||
node {
|
|
||||||
calculator: "EndLoopIntegersCalculator"
|
|
||||||
input_stream: "ITEM:int_plus_two"
|
|
||||||
input_stream: "BATCH_END:timestamp"
|
|
||||||
output_stream: "ITERABLE:ints_plus_two"
|
|
||||||
}
|
|
||||||
)pb");
|
|
||||||
std::vector<Packet> int_plus_one_packets;
|
|
||||||
tool::AddVectorSink("ints_plus_one", &graph_config, &int_plus_one_packets);
|
|
||||||
std::vector<Packet> int_original_packets;
|
|
||||||
tool::AddVectorSink("ints_plus_two", &graph_config, &int_original_packets);
|
|
||||||
|
|
||||||
CalculatorGraph graph;
|
|
||||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
|
||||||
MP_ASSERT_OK(graph.StartRun({}));
|
|
||||||
for (int i = 0; i < 100; ++i) {
|
|
||||||
std::vector<int> ints = {i, i + 1, i + 2};
|
|
||||||
Timestamp ts = Timestamp(i);
|
|
||||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
|
||||||
"ints", MakePacket<std::vector<int>>(std::move(ints)).At(ts)));
|
|
||||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
|
||||||
EXPECT_THAT(int_plus_one_packets,
|
|
||||||
testing::ElementsAre(
|
|
||||||
PacketOfIntsEq(ts, std::vector<int>{i + 1, i + 2, i + 3})));
|
|
||||||
EXPECT_THAT(int_original_packets,
|
|
||||||
testing::ElementsAre(
|
|
||||||
PacketOfIntsEq(ts, std::vector<int>{i + 2, i + 3, i + 4})));
|
|
||||||
|
|
||||||
int_plus_one_packets.clear();
|
|
||||||
int_original_packets.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_ASSERT_OK(graph.CloseAllPacketSources());
|
|
||||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Passes non empty vector through or outputs empty vector in case of timestamp
|
// Passes non empty vector through or outputs empty vector in case of timestamp
|
||||||
// bound update.
|
// bound update.
|
||||||
class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
|
class PassThroughOrEmptyVectorCalculator : public CalculatorBase {
|
||||||
|
|
|
@ -92,7 +92,7 @@ class BypassCalculator : public Node {
|
||||||
auto options = cc->Options<BypassCalculatorOptions>();
|
auto options = cc->Options<BypassCalculatorOptions>();
|
||||||
RET_CHECK_EQ(options.pass_input_stream().size(),
|
RET_CHECK_EQ(options.pass_input_stream().size(),
|
||||||
options.pass_output_stream().size());
|
options.pass_output_stream().size());
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
auto pass_streams,
|
auto pass_streams,
|
||||||
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
||||||
std::set<CollectionItemId> pass_out;
|
std::set<CollectionItemId> pass_out;
|
||||||
|
@ -121,9 +121,8 @@ class BypassCalculator : public Node {
|
||||||
// Saves the map of passthrough input and output stream ids.
|
// Saves the map of passthrough input and output stream ids.
|
||||||
absl::Status Open(CalculatorContext* cc) override {
|
absl::Status Open(CalculatorContext* cc) override {
|
||||||
auto options = cc->Options<BypassCalculatorOptions>();
|
auto options = cc->Options<BypassCalculatorOptions>();
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(pass_streams_, GetPassMap(options, *cc->Inputs().TagMap(),
|
||||||
pass_streams_,
|
*cc->Outputs().TagMap()));
|
||||||
GetPassMap(options, *cc->Inputs().TagMap(), *cc->Outputs().TagMap()));
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
// Copyright 2023 The MediaPipe Authors.
|
// Copyright 2019 The MediaPipe Authors.
|
||||||
//
|
//
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
// you may not use this file except in compliance with the License.
|
// you may not use this file except in compliance with the License.
|
||||||
|
@ -12,16 +12,15 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#import <Foundation/Foundation.h>
|
#include <vector>
|
||||||
|
|
||||||
#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h"
|
#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
|
||||||
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
|
|
||||||
NS_ASSUME_NONNULL_BEGIN
|
namespace mediapipe {
|
||||||
|
|
||||||
/** Options for setting up a `FaceStylizer`. */
|
typedef ClipVectorSizeCalculator<::mediapipe::Detection>
|
||||||
NS_SWIFT_NAME(FaceStylizerOptions)
|
ClipDetectionVectorSizeCalculator;
|
||||||
@interface MPPFaceStylizerOptions : MPPTaskOptions <NSCopying>
|
REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
|
||||||
|
|
||||||
@end
|
} // namespace mediapipe
|
||||||
|
|
||||||
NS_ASSUME_NONNULL_END
|
|
|
@ -18,7 +18,6 @@
|
||||||
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
#include "mediapipe/calculators/core/concatenate_vector_calculator.pb.h"
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/body_rig.pb.h"
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/port/canonical_errors.h"
|
#include "mediapipe/framework/port/canonical_errors.h"
|
||||||
|
@ -129,19 +128,6 @@ class ConcatenateClassificationListCalculator
|
||||||
};
|
};
|
||||||
MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
|
MEDIAPIPE_REGISTER_NODE(ConcatenateClassificationListCalculator);
|
||||||
|
|
||||||
class ConcatenateJointListCalculator
|
|
||||||
: public ConcatenateListsCalculator<Joint, JointList> {
|
|
||||||
protected:
|
|
||||||
int ListSize(const JointList& list) const override {
|
|
||||||
return list.joint_size();
|
|
||||||
}
|
|
||||||
const Joint GetItem(const JointList& list, int idx) const override {
|
|
||||||
return list.joint(idx);
|
|
||||||
}
|
|
||||||
Joint* AddItem(JointList& list) const override { return list.add_joint(); }
|
|
||||||
};
|
|
||||||
MEDIAPIPE_REGISTER_NODE(ConcatenateJointListCalculator);
|
|
||||||
|
|
||||||
} // namespace api2
|
} // namespace api2
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
||||||
|
|
|
@ -14,8 +14,6 @@
|
||||||
|
|
||||||
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
#include "mediapipe/calculators/core/end_loop_calculator.h"
|
||||||
|
|
||||||
#include <array>
|
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
|
@ -86,8 +84,4 @@ typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
|
||||||
EndLoopAffineMatrixCalculator;
|
EndLoopAffineMatrixCalculator;
|
||||||
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
|
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
|
||||||
|
|
||||||
typedef EndLoopCalculator<std::vector<std::pair<int, int>>>
|
|
||||||
EndLoopImageSizeCalculator;
|
|
||||||
REGISTER_CALCULATOR(EndLoopImageSizeCalculator);
|
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
|
@ -55,16 +55,16 @@ class EndLoopCalculator : public CalculatorBase {
|
||||||
if (!input_stream_collection_) {
|
if (!input_stream_collection_) {
|
||||||
input_stream_collection_.reset(new IterableT);
|
input_stream_collection_.reset(new IterableT);
|
||||||
}
|
}
|
||||||
|
// Try to consume the item and move it into the collection. If the items
|
||||||
if constexpr (std::is_copy_constructible_v<ItemT>) {
|
// are not consumable, then try to copy them instead. If the items are
|
||||||
input_stream_collection_->push_back(
|
// not copyable, then an error will be returned.
|
||||||
cc->Inputs().Tag("ITEM").Get<ItemT>());
|
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
||||||
|
if (item_ptr_or.ok()) {
|
||||||
|
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
||||||
} else {
|
} else {
|
||||||
// Try to consume the item and move it into the collection. Return an
|
if constexpr (std::is_copy_constructible_v<ItemT>) {
|
||||||
// error if the items are not consumable.
|
input_stream_collection_->push_back(
|
||||||
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
|
cc->Inputs().Tag("ITEM").template Get<ItemT>());
|
||||||
if (item_ptr_or.ok()) {
|
|
||||||
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
|
|
||||||
} else {
|
} else {
|
||||||
return absl::InternalError(
|
return absl::InternalError(
|
||||||
"The item type is not copiable. Consider making the "
|
"The item type is not copiable. Consider making the "
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
#include "mediapipe/framework/port/gtest.h"
|
#include "mediapipe/framework/port/gtest.h"
|
||||||
|
@ -357,18 +356,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
|
||||||
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
|
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
|
||||||
constexpr int64_t kTimestampValue3 = 45;
|
constexpr int64_t kTimestampValue3 = 45;
|
||||||
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
|
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
|
||||||
ABSL_LOG(INFO) << "a";
|
LOG(INFO) << "a";
|
||||||
const std::vector<Packet>& output =
|
const std::vector<Packet>& output =
|
||||||
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
|
||||||
ABSL_LOG(INFO) << "s";
|
LOG(INFO) << "s";
|
||||||
ASSERT_EQ(2, output.size());
|
ASSERT_EQ(2, output.size());
|
||||||
ABSL_LOG(INFO) << "d";
|
LOG(INFO) << "d";
|
||||||
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
|
||||||
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
|
||||||
ABSL_LOG(INFO) << "f";
|
LOG(INFO) << "f";
|
||||||
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
|
||||||
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
|
||||||
ABSL_LOG(INFO) << "g";
|
LOG(INFO) << "g";
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
|
@ -79,7 +78,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
|
||||||
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
|
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
|
||||||
cc->Outputs().Index(0).AddPacket(packet);
|
cc->Outputs().Index(0).AddPacket(packet);
|
||||||
} else {
|
} else {
|
||||||
ABSL_LOG_FIRST_N(WARNING, 5)
|
LOG_FIRST_N(WARNING, 5)
|
||||||
<< "Dropping a packet with timestamp " << packet.Timestamp();
|
<< "Dropping a packet with timestamp " << packet.Timestamp();
|
||||||
}
|
}
|
||||||
if (cc->Outputs().NumEntries() >= 2) {
|
if (cc->Outputs().NumEntries() >= 2) {
|
||||||
|
|
|
@ -16,7 +16,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
#include "mediapipe/framework/formats/matrix.h"
|
#include "mediapipe/framework/formats/matrix.h"
|
||||||
|
@ -210,7 +209,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
|
||||||
MatrixFromTextProto(kSamplesText, &samples);
|
MatrixFromTextProto(kSamplesText, &samples);
|
||||||
Matrix expected;
|
Matrix expected;
|
||||||
MatrixFromTextProto(kExpectedText, &expected);
|
MatrixFromTextProto(kExpectedText, &expected);
|
||||||
ABSL_CHECK_EQ(samples.cols(), expected.cols());
|
CHECK_EQ(samples.cols(), expected.cols());
|
||||||
|
|
||||||
for (int i = 0; i < samples.cols(); ++i) {
|
for (int i = 0; i < samples.cols(); ++i) {
|
||||||
// Take a column from samples and produce a packet with just that
|
// Take a column from samples and produce a packet with just that
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
@ -54,7 +53,7 @@ class MergeCalculator : public Node {
|
||||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||||
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
|
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
|
||||||
if (kIn(cc).Count() == 1) {
|
if (kIn(cc).Count() == 1) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING)
|
||||||
<< "MergeCalculator expects multiple input streams to merge but is "
|
<< "MergeCalculator expects multiple input streams to merge but is "
|
||||||
"receiving only one. Make sure the calculator is configured "
|
"receiving only one. Make sure the calculator is configured "
|
||||||
"correctly or consider removing this calculator to reduce "
|
"correctly or consider removing this calculator to reduce "
|
||||||
|
@ -73,8 +72,8 @@ class MergeCalculator : public Node {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ABSL_LOG(WARNING) << "Empty input packets at timestamp "
|
LOG(WARNING) << "Empty input packets at timestamp "
|
||||||
<< cc->InputTimestamp().Value();
|
<< cc->InputTimestamp().Value();
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,9 +16,6 @@
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
// Reflect an integer against the lower and upper bound of an interval.
|
// Reflect an integer against the lower and upper bound of an interval.
|
||||||
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
|
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
|
||||||
|
@ -180,7 +177,7 @@ PacketResamplerCalculator::GetSamplingStrategy(
|
||||||
const PacketResamplerCalculatorOptions& options) {
|
const PacketResamplerCalculatorOptions& options) {
|
||||||
if (options.reproducible_sampling()) {
|
if (options.reproducible_sampling()) {
|
||||||
if (!options.jitter_with_reflection()) {
|
if (!options.jitter_with_reflection()) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING)
|
||||||
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
<< "reproducible_sampling enabled w/ jitter_with_reflection "
|
||||||
"disabled. "
|
"disabled. "
|
||||||
<< "reproducible_sampling always uses jitter with reflection, "
|
<< "reproducible_sampling always uses jitter with reflection, "
|
||||||
|
@ -203,15 +200,15 @@ PacketResamplerCalculator::GetSamplingStrategy(
|
||||||
|
|
||||||
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
|
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
|
||||||
int64_t index) const {
|
int64_t index) const {
|
||||||
ABSL_CHECK_EQ(jitter_, 0.0);
|
CHECK_EQ(jitter_, 0.0);
|
||||||
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
|
CHECK_NE(first_timestamp_, Timestamp::Unset());
|
||||||
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
|
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
|
||||||
}
|
}
|
||||||
|
|
||||||
int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
|
int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
|
||||||
Timestamp timestamp) const {
|
Timestamp timestamp) const {
|
||||||
ABSL_CHECK_EQ(jitter_, 0.0);
|
CHECK_EQ(jitter_, 0.0);
|
||||||
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
|
CHECK_NE(first_timestamp_, Timestamp::Unset());
|
||||||
return MathUtil::SafeRound<int64_t, double>(
|
return MathUtil::SafeRound<int64_t, double>(
|
||||||
(timestamp - first_timestamp_).Seconds() * frame_rate_);
|
(timestamp - first_timestamp_).Seconds() * frame_rate_);
|
||||||
}
|
}
|
||||||
|
@ -232,15 +229,13 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||||
<< "VideoHeader::frame_rate holds the target value and not "
|
"the actual value.";
|
||||||
"the actual value.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
"ignored, because we are adding jitter.";
|
||||||
"ignored, because we are adding jitter.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
|
@ -259,7 +254,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
|
||||||
}
|
}
|
||||||
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
|
||||||
if (!packet_reservoir_->IsEmpty()) {
|
if (!packet_reservoir_->IsEmpty()) {
|
||||||
ABSL_LOG(INFO) << "Emitting pack from reservoir.";
|
LOG(INFO) << "Emitting pack from reservoir.";
|
||||||
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -290,7 +285,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Process(
|
||||||
|
|
||||||
if (calculator_->frame_time_usec_ <
|
if (calculator_->frame_time_usec_ <
|
||||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||||
ABSL_LOG_FIRST_N(WARNING, 2)
|
LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -345,8 +340,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
|
||||||
next_output_timestamp_ = Timestamp(ReflectBetween(
|
next_output_timestamp_ = Timestamp(ReflectBetween(
|
||||||
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
|
||||||
next_output_timestamp_max_.Value()));
|
next_output_timestamp_max_.Value()));
|
||||||
ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
|
||||||
ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||||
|
@ -357,15 +352,13 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
|
||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||||
<< "VideoHeader::frame_rate holds the target value and not "
|
"the actual value.";
|
||||||
"the actual value.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
"ignored, because we are adding jitter.";
|
||||||
"ignored, because we are adding jitter.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
|
@ -418,7 +411,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process(
|
||||||
// Note, if the stream is upsampling, this could lead to the same packet
|
// Note, if the stream is upsampling, this could lead to the same packet
|
||||||
// being emitted twice. Upsampling and jitter doesn't make much sense
|
// being emitted twice. Upsampling and jitter doesn't make much sense
|
||||||
// but does technically work.
|
// but does technically work.
|
||||||
ABSL_LOG_FIRST_N(WARNING, 2)
|
LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -506,15 +499,13 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
if (resampler_options.output_header() !=
|
if (resampler_options.output_header() !=
|
||||||
PacketResamplerCalculatorOptions::NONE) {
|
PacketResamplerCalculatorOptions::NONE) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
|
||||||
<< "VideoHeader::frame_rate holds the target value and not "
|
"the actual value.";
|
||||||
"the actual value.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (calculator_->flush_last_packet_) {
|
if (calculator_->flush_last_packet_) {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
|
||||||
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
|
"ignored, because we are adding jitter.";
|
||||||
"ignored, because we are adding jitter.";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
|
||||||
|
@ -564,7 +555,7 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
|
||||||
|
|
||||||
if (calculator_->frame_time_usec_ <
|
if (calculator_->frame_time_usec_ <
|
||||||
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
|
||||||
ABSL_LOG_FIRST_N(WARNING, 2)
|
LOG_FIRST_N(WARNING, 2)
|
||||||
<< "Adding jitter is not very useful when upsampling.";
|
<< "Adding jitter is not very useful when upsampling.";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,6 +13,7 @@
|
||||||
#include "mediapipe/framework/deps/random_base.h"
|
#include "mediapipe/framework/deps/random_base.h"
|
||||||
#include "mediapipe/framework/formats/video_stream_header.h"
|
#include "mediapipe/framework/formats/video_stream_header.h"
|
||||||
#include "mediapipe/framework/port/integral_types.h"
|
#include "mediapipe/framework/port/integral_types.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/port/status_macros.h"
|
#include "mediapipe/framework/port/status_macros.h"
|
||||||
|
|
|
@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
|
||||||
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
|
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shows how control packets receive timestamps before and after frame packets
|
// Shows how control packets recieve timestamps before and after frame packets
|
||||||
// have arrived.
|
// have arrived.
|
||||||
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
|
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
|
||||||
CalculatorGraphConfig::Node node_config = BuildNodeConfig();
|
CalculatorGraphConfig::Node node_config = BuildNodeConfig();
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
#include <cmath> // for ceil
|
#include <cmath> // for ceil
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_context.h"
|
#include "mediapipe/framework/calculator_context.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -161,8 +160,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
thinner_type_ = options.thinner_type();
|
thinner_type_ = options.thinner_type();
|
||||||
// This check enables us to assume only two thinner types exist in Process()
|
// This check enables us to assume only two thinner types exist in Process()
|
||||||
ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
|
CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
|
||||||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
|
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
|
||||||
<< "Unsupported thinner type.";
|
<< "Unsupported thinner type.";
|
||||||
|
|
||||||
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
|
||||||
|
@ -178,8 +177,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
||||||
} else {
|
} else {
|
||||||
period_ = TimestampDiff(options.period());
|
period_ = TimestampDiff(options.period());
|
||||||
}
|
}
|
||||||
ABSL_CHECK_LT(TimestampDiff(0), period_)
|
CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
|
||||||
<< "Specified period must be positive.";
|
|
||||||
|
|
||||||
if (options.has_start_time()) {
|
if (options.has_start_time()) {
|
||||||
start_time_ = Timestamp(options.start_time());
|
start_time_ = Timestamp(options.start_time());
|
||||||
|
@ -191,7 +189,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
end_time_ =
|
end_time_ =
|
||||||
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
|
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
|
||||||
ABSL_CHECK_LT(start_time_, end_time_)
|
CHECK_LT(start_time_, end_time_)
|
||||||
<< "Invalid PacketThinner: start_time must be earlier than end_time";
|
<< "Invalid PacketThinner: start_time must be earlier than end_time";
|
||||||
|
|
||||||
sync_output_timestamps_ = options.sync_output_timestamps();
|
sync_output_timestamps_ = options.sync_output_timestamps();
|
||||||
|
@ -234,7 +232,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
|
||||||
// Emit any saved packets before quitting.
|
// Emit any saved packets before quitting.
|
||||||
if (!saved_packet_.IsEmpty()) {
|
if (!saved_packet_.IsEmpty()) {
|
||||||
// Only sync thinner should have saved packets.
|
// Only sync thinner should have saved packets.
|
||||||
ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
|
CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
|
||||||
if (sync_output_timestamps_) {
|
if (sync_output_timestamps_) {
|
||||||
cc->Outputs().Index(0).AddPacket(
|
cc->Outputs().Index(0).AddPacket(
|
||||||
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
|
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
|
||||||
|
@ -271,7 +269,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
|
||||||
const Timestamp saved_sync = NearestSyncTimestamp(saved);
|
const Timestamp saved_sync = NearestSyncTimestamp(saved);
|
||||||
const Timestamp now = cc->InputTimestamp();
|
const Timestamp now = cc->InputTimestamp();
|
||||||
const Timestamp now_sync = NearestSyncTimestamp(now);
|
const Timestamp now_sync = NearestSyncTimestamp(now);
|
||||||
ABSL_CHECK_LE(saved_sync, now_sync);
|
CHECK_LE(saved_sync, now_sync);
|
||||||
if (saved_sync == now_sync) {
|
if (saved_sync == now_sync) {
|
||||||
// Saved Packet is in same interval as current packet.
|
// Saved Packet is in same interval as current packet.
|
||||||
// Replace saved packet with current if it is at least as
|
// Replace saved packet with current if it is at least as
|
||||||
|
@ -297,7 +295,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
|
||||||
}
|
}
|
||||||
|
|
||||||
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
||||||
ABSL_CHECK_NE(start_time_, Timestamp::Unset())
|
CHECK_NE(start_time_, Timestamp::Unset())
|
||||||
<< "Method only valid for sync thinner calculator.";
|
<< "Method only valid for sync thinner calculator.";
|
||||||
|
|
||||||
// Computation is done using int64 arithmetic. No easy way to avoid
|
// Computation is done using int64 arithmetic. No easy way to avoid
|
||||||
|
@ -305,12 +303,12 @@ Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
|
||||||
const int64_t now64 = now.Value();
|
const int64_t now64 = now.Value();
|
||||||
const int64_t start64 = start_time_.Value();
|
const int64_t start64 = start_time_.Value();
|
||||||
const int64_t period64 = period_.Value();
|
const int64_t period64 = period_.Value();
|
||||||
ABSL_CHECK_LE(0, period64);
|
CHECK_LE(0, period64);
|
||||||
|
|
||||||
// Round now64 to its closest interval (units of period64).
|
// Round now64 to its closest interval (units of period64).
|
||||||
int64_t sync64 =
|
int64_t sync64 =
|
||||||
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
|
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
|
||||||
ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2)
|
CHECK_LE(abs(now64 - sync64), period64 / 2)
|
||||||
<< "start64: " << start64 << "; now64: " << now64
|
<< "start64: " << start64 << "; now64: " << now64
|
||||||
<< "; sync64: " << sync64;
|
<< "; sync64: " << sync64;
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,6 @@
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -71,7 +70,7 @@ class SimpleRunner : public CalculatorRunner {
|
||||||
}
|
}
|
||||||
|
|
||||||
double GetFrameRate() const {
|
double GetFrameRate() const {
|
||||||
ABSL_CHECK(!Outputs().Index(0).header.IsEmpty());
|
CHECK(!Outputs().Index(0).header.IsEmpty());
|
||||||
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
|
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
|
|
||||||
#include <deque>
|
#include <deque>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
|
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
|
||||||
#include "mediapipe/framework/api2/node.h"
|
#include "mediapipe/framework/api2/node.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -102,7 +101,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
|
||||||
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
|
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
|
||||||
packet_cache_.pop_front();
|
packet_cache_.pop_front();
|
||||||
} else if (emit_empty_packets_before_first_packet_) {
|
} else if (emit_empty_packets_before_first_packet_) {
|
||||||
ABSL_LOG(FATAL) << "Not supported yet";
|
LOG(FATAL) << "Not supported yet";
|
||||||
}
|
}
|
||||||
// Store current packet for later output.
|
// Store current packet for later output.
|
||||||
packet_cache_.push_back(kIn(cc).packet());
|
packet_cache_.push_back(kIn(cc).packet());
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
#include <set>
|
#include <set>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/status/status.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
@ -33,7 +32,6 @@ namespace {
|
||||||
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
|
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
|
||||||
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
|
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
|
||||||
constexpr char kTagAtZero[] = "AT_ZERO";
|
constexpr char kTagAtZero[] = "AT_ZERO";
|
||||||
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
|
|
||||||
constexpr char kTagAtTick[] = "AT_TICK";
|
constexpr char kTagAtTick[] = "AT_TICK";
|
||||||
constexpr char kTagTick[] = "TICK";
|
constexpr char kTagTick[] = "TICK";
|
||||||
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
|
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
|
||||||
|
@ -45,7 +43,6 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
|
||||||
res->emplace(kTagAtPostStream, Timestamp::PostStream());
|
res->emplace(kTagAtPostStream, Timestamp::PostStream());
|
||||||
res->emplace(kTagAtZero, Timestamp(0));
|
res->emplace(kTagAtZero, Timestamp(0));
|
||||||
res->emplace(kTagAtTick, Timestamp::Unset());
|
res->emplace(kTagAtTick, Timestamp::Unset());
|
||||||
res->emplace(kTagAtFirstTick, Timestamp::Unset());
|
|
||||||
res->emplace(kTagAtTimestamp, Timestamp::Unset());
|
res->emplace(kTagAtTimestamp, Timestamp::Unset());
|
||||||
return res;
|
return res;
|
||||||
}();
|
}();
|
||||||
|
@ -62,8 +59,8 @@ std::string GetOutputTag(const CC& cc) {
|
||||||
// timestamp, depending on the tag used to define output stream(s). (One tag can
|
// timestamp, depending on the tag used to define output stream(s). (One tag can
|
||||||
// be used only.)
|
// be used only.)
|
||||||
//
|
//
|
||||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
|
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
|
||||||
// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
|
// and corresponding timestamps are Timestamp::PreStream(),
|
||||||
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
|
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
|
||||||
// input, and timestamp received from a side input.
|
// input, and timestamp received from a side input.
|
||||||
//
|
//
|
||||||
|
@ -99,7 +96,6 @@ class SidePacketToStreamCalculator : public CalculatorBase {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
bool is_tick_processing_ = false;
|
bool is_tick_processing_ = false;
|
||||||
bool close_on_first_tick_ = false;
|
|
||||||
std::string output_tag_;
|
std::string output_tag_;
|
||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||||
|
@ -107,16 +103,13 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||||
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
||||||
const auto& tags = cc->Outputs().GetTags();
|
const auto& tags = cc->Outputs().GetTags();
|
||||||
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
|
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
|
||||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
|
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||||
"AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
|
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||||
"specify output stream(s).";
|
"stream(s).";
|
||||||
const bool has_tick_output =
|
RET_CHECK(
|
||||||
cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
|
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
|
||||||
const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
|
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
|
||||||
RET_CHECK((has_tick_output && has_tick_input) ||
|
<< "Either both of TICK and AT_TICK should be used or none of them.";
|
||||||
(!has_tick_output && !has_tick_input))
|
|
||||||
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
|
||||||
"should be used or none of them.";
|
|
||||||
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
|
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||||
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
|
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
|
||||||
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
|
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||||
|
@ -155,17 +148,11 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
|
||||||
// timestamp bound update.
|
// timestamp bound update.
|
||||||
cc->SetOffset(TimestampDiff(0));
|
cc->SetOffset(TimestampDiff(0));
|
||||||
}
|
}
|
||||||
if (output_tag_ == kTagAtFirstTick) {
|
|
||||||
close_on_first_tick_ = true;
|
|
||||||
}
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
||||||
if (is_tick_processing_) {
|
if (is_tick_processing_) {
|
||||||
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
|
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
// TICK input is guaranteed to be non-empty, as it's the only input stream
|
// TICK input is guaranteed to be non-empty, as it's the only input stream
|
||||||
// for this calculator.
|
// for this calculator.
|
||||||
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
|
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
|
||||||
|
@ -173,9 +160,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
||||||
cc->Outputs()
|
cc->Outputs()
|
||||||
.Get(output_tag_, i)
|
.Get(output_tag_, i)
|
||||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||||
if (close_on_first_tick_) {
|
|
||||||
cc->Outputs().Get(output_tag_, i).Close();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -186,7 +170,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
||||||
|
|
||||||
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||||
if (!cc->Outputs().HasTag(kTagAtTick) &&
|
if (!cc->Outputs().HasTag(kTagAtTick) &&
|
||||||
!cc->Outputs().HasTag(kTagAtFirstTick) &&
|
|
||||||
!cc->Outputs().HasTag(kTagAtTimestamp)) {
|
!cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||||
const auto& timestamp = kTimestampMap->at(output_tag_);
|
const auto& timestamp = kTimestampMap->at(output_tag_);
|
||||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||||
|
|
|
@ -27,17 +27,13 @@
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/port/status_matchers.h"
|
#include "mediapipe/framework/port/status_matchers.h"
|
||||||
#include "mediapipe/framework/tool/options_util.h"
|
#include "mediapipe/framework/tool/options_util.h"
|
||||||
#include "mediapipe/util/packet_test_util.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
using ::testing::ElementsAre;
|
using testing::HasSubstr;
|
||||||
using ::testing::Eq;
|
|
||||||
using ::testing::HasSubstr;
|
|
||||||
using ::testing::IsEmpty;
|
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
|
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -56,35 +52,10 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
|
||||||
EXPECT_THAT(
|
EXPECT_THAT(
|
||||||
status.message(),
|
status.message(),
|
||||||
HasSubstr(
|
HasSubstr(
|
||||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
"Either both of TICK and AT_TICK should be used or none of them."));
|
||||||
"should be used or none of them."));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator,
|
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||||
WrongConfigWithMissingTickForFirstTickProcessing) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
R"pb(
|
|
||||||
input_stream: "tick"
|
|
||||||
input_side_packet: "side_packet"
|
|
||||||
output_stream: "packet"
|
|
||||||
node {
|
|
||||||
calculator: "SidePacketToStreamCalculator"
|
|
||||||
input_side_packet: "side_packet"
|
|
||||||
output_stream: "AT_FIRST_TICK:packet"
|
|
||||||
}
|
|
||||||
)pb");
|
|
||||||
CalculatorGraph graph;
|
|
||||||
auto status = graph.Initialize(graph_config);
|
|
||||||
EXPECT_FALSE(status.ok());
|
|
||||||
EXPECT_THAT(
|
|
||||||
status.message(),
|
|
||||||
HasSubstr(
|
|
||||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
|
||||||
"should be used or none of them."));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -105,7 +76,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
|
||||||
"or none of them."));
|
"or none of them."));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
|
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -121,13 +92,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
|
||||||
CalculatorGraph graph;
|
CalculatorGraph graph;
|
||||||
auto status = graph.Initialize(graph_config);
|
auto status = graph.Initialize(graph_config);
|
||||||
EXPECT_FALSE(status.ok());
|
EXPECT_FALSE(status.ok());
|
||||||
EXPECT_THAT(status.message(),
|
EXPECT_THAT(
|
||||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
status.message(),
|
||||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||||
"allowed and required to specify output stream(s)."));
|
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||||
|
"stream(s)."));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
|
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -145,13 +117,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
|
||||||
CalculatorGraph graph;
|
CalculatorGraph graph;
|
||||||
auto status = graph.Initialize(graph_config);
|
auto status = graph.Initialize(graph_config);
|
||||||
EXPECT_FALSE(status.ok());
|
EXPECT_FALSE(status.ok());
|
||||||
EXPECT_THAT(status.message(),
|
EXPECT_THAT(
|
||||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
status.message(),
|
||||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||||
"allowed and required to specify output stream(s)."));
|
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||||
|
"stream(s)."));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
|
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -173,7 +146,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
|
||||||
"Same number of input side packets and output streams is required."));
|
"Same number of input side packets and output streams is required."));
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
|
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -275,50 +248,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
||||||
tick_and_verify(/*at_timestamp=*/1025);
|
tick_and_verify(/*at_timestamp=*/1025);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, AtFirstTick) {
|
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
R"pb(
|
|
||||||
input_stream: "tick"
|
|
||||||
input_side_packet: "side_packet"
|
|
||||||
output_stream: "packet"
|
|
||||||
node {
|
|
||||||
calculator: "SidePacketToStreamCalculator"
|
|
||||||
input_stream: "TICK:tick"
|
|
||||||
input_side_packet: "side_packet"
|
|
||||||
output_stream: "AT_FIRST_TICK:packet"
|
|
||||||
}
|
|
||||||
)pb");
|
|
||||||
std::vector<Packet> output_packets;
|
|
||||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
|
||||||
CalculatorGraph graph;
|
|
||||||
|
|
||||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
|
||||||
const int expected_value = 20;
|
|
||||||
const Timestamp kTestTimestamp(1234);
|
|
||||||
MP_ASSERT_OK(
|
|
||||||
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
|
|
||||||
|
|
||||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
|
||||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
|
||||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
|
||||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
|
||||||
};
|
|
||||||
|
|
||||||
insert_tick(kTestTimestamp);
|
|
||||||
|
|
||||||
EXPECT_THAT(output_packets,
|
|
||||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
|
||||||
Eq(kTestTimestamp), Eq(expected_value))));
|
|
||||||
|
|
||||||
output_packets.clear();
|
|
||||||
|
|
||||||
// Should not result in an additional output.
|
|
||||||
insert_tick(kTestTimestamp + 1);
|
|
||||||
EXPECT_THAT(output_packets, IsEmpty());
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
@ -372,62 +302,6 @@ TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
|
||||||
tick_and_verify(/*at_timestamp=*/1025);
|
tick_and_verify(/*at_timestamp=*/1025);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
R"pb(
|
|
||||||
input_stream: "tick"
|
|
||||||
input_side_packet: "side_packet0"
|
|
||||||
input_side_packet: "side_packet1"
|
|
||||||
output_stream: "packet0"
|
|
||||||
output_stream: "packet1"
|
|
||||||
node {
|
|
||||||
calculator: "SidePacketToStreamCalculator"
|
|
||||||
input_stream: "TICK:tick"
|
|
||||||
input_side_packet: "side_packet0"
|
|
||||||
input_side_packet: "side_packet1"
|
|
||||||
output_stream: "AT_FIRST_TICK:0:packet0"
|
|
||||||
output_stream: "AT_FIRST_TICK:1:packet1"
|
|
||||||
}
|
|
||||||
)pb");
|
|
||||||
std::vector<Packet> output_packets0;
|
|
||||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
|
||||||
std::vector<Packet> output_packets1;
|
|
||||||
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
|
|
||||||
CalculatorGraph graph;
|
|
||||||
|
|
||||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
|
||||||
const int expected_value0 = 20;
|
|
||||||
const int expected_value1 = 128;
|
|
||||||
const Timestamp kTestTimestamp(1234);
|
|
||||||
MP_ASSERT_OK(
|
|
||||||
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
|
|
||||||
{"side_packet1", MakePacket<int>(expected_value1)}}));
|
|
||||||
|
|
||||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
|
||||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
|
||||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
|
||||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
|
||||||
};
|
|
||||||
|
|
||||||
insert_tick(kTestTimestamp);
|
|
||||||
|
|
||||||
EXPECT_THAT(output_packets0,
|
|
||||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
|
||||||
Eq(kTestTimestamp), Eq(expected_value0))));
|
|
||||||
EXPECT_THAT(output_packets1,
|
|
||||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
|
||||||
Eq(kTestTimestamp), Eq(expected_value1))));
|
|
||||||
|
|
||||||
output_packets0.clear();
|
|
||||||
output_packets1.clear();
|
|
||||||
|
|
||||||
// Should not result in an additional output.
|
|
||||||
insert_tick(kTestTimestamp + 1);
|
|
||||||
EXPECT_THAT(output_packets0, IsEmpty());
|
|
||||||
EXPECT_THAT(output_packets1, IsEmpty());
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
|
@ -460,7 +334,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||||
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
|
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||||
CalculatorGraphConfig graph_config =
|
CalculatorGraphConfig graph_config =
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||||
R"pb(
|
R"pb(
|
||||||
|
|
|
@ -17,7 +17,6 @@
|
||||||
|
|
||||||
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/body_rig.pb.h"
|
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/port/canonical_errors.h"
|
#include "mediapipe/framework/port/canonical_errors.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
@ -197,18 +196,6 @@ class SplitLandmarkListCalculator
|
||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(SplitLandmarkListCalculator);
|
REGISTER_CALCULATOR(SplitLandmarkListCalculator);
|
||||||
|
|
||||||
class SplitJointListCalculator : public SplitListsCalculator<Joint, JointList> {
|
|
||||||
protected:
|
|
||||||
int ListSize(const JointList& list) const override {
|
|
||||||
return list.joint_size();
|
|
||||||
}
|
|
||||||
const Joint GetItem(const JointList& list, int idx) const override {
|
|
||||||
return list.joint(idx);
|
|
||||||
}
|
|
||||||
Joint* AddItem(JointList& list) const override { return list.add_joint(); }
|
|
||||||
};
|
|
||||||
REGISTER_CALCULATOR(SplitJointListCalculator);
|
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
||||||
// NOLINTNEXTLINE
|
// NOLINTNEXTLINE
|
||||||
|
|
|
@ -1,90 +0,0 @@
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
|
||||||
#include "mediapipe/framework/port/status.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
constexpr char kInputValueTag[] = "IN";
|
|
||||||
constexpr char kTickerTag[] = "TICK";
|
|
||||||
constexpr char kOutputTag[] = "OUT";
|
|
||||||
constexpr char kIndicationTag[] = "FLAG";
|
|
||||||
|
|
||||||
} // namespace
|
|
||||||
// For every packet received on the TICK stream, if the IN stream is not
|
|
||||||
// empty - emit its value as is as OUT. Otherwise output a default packet.
|
|
||||||
// FLAG outputs true every time the default value has been used. It does not
|
|
||||||
// output anything when IN has a value.
|
|
||||||
//
|
|
||||||
// Example config:
|
|
||||||
// node {
|
|
||||||
// calculator: "ValueOrDefaultCalculator"
|
|
||||||
// input_stream: "IN:sometimes_missing_value"
|
|
||||||
// input_stream: "TICK:clock"
|
|
||||||
// output_stream: "OUT:value_or_default"
|
|
||||||
// output_stream: "FLAG:used_default"
|
|
||||||
// input_side_packet: "default"
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// TODO: Consider adding an option for a default value as a input-stream
|
|
||||||
// instead of a side-packet, so it will enable using standard calculators
|
|
||||||
// instead of creating a new packet-generators. It will also allow a dynamic
|
|
||||||
// default value.
|
|
||||||
class ValueOrDefaultCalculator : public mediapipe::CalculatorBase {
|
|
||||||
public:
|
|
||||||
ValueOrDefaultCalculator() {}
|
|
||||||
|
|
||||||
ValueOrDefaultCalculator(const ValueOrDefaultCalculator&) = delete;
|
|
||||||
ValueOrDefaultCalculator& operator=(const ValueOrDefaultCalculator&) = delete;
|
|
||||||
|
|
||||||
static mediapipe::Status GetContract(mediapipe::CalculatorContract* cc) {
|
|
||||||
cc->Inputs().Tag(kInputValueTag).SetAny();
|
|
||||||
cc->Inputs().Tag(kTickerTag).SetAny();
|
|
||||||
cc->Outputs().Tag(kOutputTag).SetSameAs(&cc->Inputs().Tag(kInputValueTag));
|
|
||||||
cc->Outputs().Tag(kIndicationTag).Set<bool>();
|
|
||||||
cc->InputSidePackets().Index(0).SetSameAs(
|
|
||||||
&cc->Inputs().Tag(kInputValueTag));
|
|
||||||
|
|
||||||
return mediapipe::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
mediapipe::Status Open(mediapipe::CalculatorContext* cc) override {
|
|
||||||
if (!cc->Inputs().Tag(kInputValueTag).Header().IsEmpty()) {
|
|
||||||
cc->Outputs()
|
|
||||||
.Tag(kOutputTag)
|
|
||||||
.SetHeader(cc->Inputs().Tag(kInputValueTag).Header());
|
|
||||||
}
|
|
||||||
default_ = cc->InputSidePackets().Index(0);
|
|
||||||
cc->SetOffset(mediapipe::TimestampDiff(0));
|
|
||||||
return mediapipe::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
mediapipe::Status Process(mediapipe::CalculatorContext* cc) override {
|
|
||||||
// Output according to the TICK signal.
|
|
||||||
if (cc->Inputs().Tag(kTickerTag).IsEmpty()) {
|
|
||||||
return mediapipe::OkStatus();
|
|
||||||
}
|
|
||||||
if (!cc->Inputs().Tag(kInputValueTag).IsEmpty()) {
|
|
||||||
// Output the input as is:
|
|
||||||
cc->Outputs()
|
|
||||||
.Tag(kOutputTag)
|
|
||||||
.AddPacket(cc->Inputs().Tag(kInputValueTag).Value());
|
|
||||||
} else {
|
|
||||||
// Output default:
|
|
||||||
cc->Outputs()
|
|
||||||
.Tag(kOutputTag)
|
|
||||||
.AddPacket(default_.At(cc->InputTimestamp()));
|
|
||||||
cc->Outputs()
|
|
||||||
.Tag(kIndicationTag)
|
|
||||||
.Add(new bool(true), cc->InputTimestamp());
|
|
||||||
}
|
|
||||||
return mediapipe::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
private:
|
|
||||||
// The default value to replicate every time there is no new value.
|
|
||||||
mediapipe::Packet default_;
|
|
||||||
};
|
|
||||||
|
|
||||||
REGISTER_CALCULATOR(ValueOrDefaultCalculator);
|
|
||||||
|
|
||||||
} // namespace mediapipe
|
|
|
@ -1,240 +0,0 @@
|
||||||
#include <algorithm>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
|
||||||
#include "mediapipe/framework/packet.h"
|
|
||||||
#include "mediapipe/framework/port/gmock.h"
|
|
||||||
#include "mediapipe/framework/port/gtest.h"
|
|
||||||
#include "mediapipe/framework/port/status_matchers.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
using ::testing::AllOf;
|
|
||||||
using ::testing::ContainerEq;
|
|
||||||
using ::testing::Each;
|
|
||||||
using ::testing::ElementsAre;
|
|
||||||
using ::testing::IsEmpty;
|
|
||||||
using ::testing::SizeIs;
|
|
||||||
using ::testing::Test;
|
|
||||||
|
|
||||||
const int kDefaultValue = 0;
|
|
||||||
|
|
||||||
// Utility to a create a mediapipe graph runner with the tested calculator and a
|
|
||||||
// default value, for all the tests.
|
|
||||||
class ValueOrDefaultRunner : public mediapipe::CalculatorRunner {
|
|
||||||
public:
|
|
||||||
ValueOrDefaultRunner()
|
|
||||||
: mediapipe::CalculatorRunner(R"pb(
|
|
||||||
calculator: "ValueOrDefaultCalculator"
|
|
||||||
input_stream: "IN:in"
|
|
||||||
input_stream: "TICK:tick"
|
|
||||||
input_side_packet: "default"
|
|
||||||
output_stream: "OUT:out"
|
|
||||||
output_stream: "FLAG:used_default"
|
|
||||||
)pb") {
|
|
||||||
MutableSidePackets()->Index(0) = mediapipe::MakePacket<int>(kDefaultValue);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Utility to push inputs to the runner to the TICK stream, so we could easily
|
|
||||||
// tick.
|
|
||||||
void TickAt(int64_t time) {
|
|
||||||
// The type or value of the stream isn't relevant, we use just a bool.
|
|
||||||
MutableInputs()->Tag("TICK").packets.push_back(
|
|
||||||
mediapipe::Adopt(new bool(false)).At(mediapipe::Timestamp(time)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Utility to push the real inputs to the runner (IN stream).
|
|
||||||
void ProvideInput(int64_t time, int value) {
|
|
||||||
MutableInputs()->Tag("IN").packets.push_back(
|
|
||||||
mediapipe::Adopt(new int(value)).At(mediapipe::Timestamp(time)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extracts the timestamps (as int64) of the output stream of the calculator.
|
|
||||||
std::vector<int64_t> GetOutputTimestamps() const {
|
|
||||||
std::vector<int64_t> timestamps;
|
|
||||||
for (const mediapipe::Packet& packet : Outputs().Tag("OUT").packets) {
|
|
||||||
timestamps.emplace_back(packet.Timestamp().Value());
|
|
||||||
}
|
|
||||||
return timestamps;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extracts the values from the output stream of the calculator.
|
|
||||||
std::vector<int> GetOutputValues() const {
|
|
||||||
std::vector<int> values;
|
|
||||||
for (const mediapipe::Packet& packet : Outputs().Tag("OUT").packets) {
|
|
||||||
values.emplace_back(packet.Get<int>());
|
|
||||||
}
|
|
||||||
return values;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extracts the timestamps (as int64) of the flag stream, which indicates on
|
|
||||||
// times without an input value (i.e. using the default value).
|
|
||||||
std::vector<int64_t> GetFlagTimestamps() const {
|
|
||||||
std::vector<int64_t> timestamps;
|
|
||||||
for (const mediapipe::Packet& packet : Outputs().Tag("FLAG").packets) {
|
|
||||||
timestamps.emplace_back(packet.Timestamp().Value());
|
|
||||||
}
|
|
||||||
return timestamps;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extracts the output from the flags stream (which should always be true).
|
|
||||||
std::vector<bool> GetFlagValues() const {
|
|
||||||
std::vector<bool> flags;
|
|
||||||
for (const mediapipe::Packet& packet : Outputs().Tag("FLAG").packets) {
|
|
||||||
flags.emplace_back(packet.Get<bool>());
|
|
||||||
}
|
|
||||||
return flags;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// To be used as input values:
|
|
||||||
std::vector<int> GetIntegersRange(int size) {
|
|
||||||
std::vector<int> result;
|
|
||||||
for (int i = 0; i < size; ++i) {
|
|
||||||
// We start with default-value+1 so it won't contain the default value.
|
|
||||||
result.push_back(kDefaultValue + 1 + i);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, NoInputs) {
|
|
||||||
// Check that when no real inputs are provided - we get the default value over
|
|
||||||
// and over, with the correct timestamps.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
const std::vector<int64_t> ticks = {0, 1, 2, 5, 8, 12, 33, 231};
|
|
||||||
|
|
||||||
for (int tick : ticks) {
|
|
||||||
runner.TickAt(tick);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
|
|
||||||
// Make sure we get the right timestamps:
|
|
||||||
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
|
||||||
// All should be default value:
|
|
||||||
EXPECT_THAT(runner.GetOutputValues(),
|
|
||||||
AllOf(Each(kDefaultValue), SizeIs(ticks.size())));
|
|
||||||
// We should get the default indication all the time:
|
|
||||||
EXPECT_THAT(runner.GetFlagTimestamps(), ContainerEq(ticks));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, NeverDefault) {
|
|
||||||
// Check that when we provide the inputs on time - we get them as outputs.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
const std::vector<int64_t> ticks = {0, 1, 2, 5, 8, 12, 33, 231};
|
|
||||||
const std::vector<int> values = GetIntegersRange(ticks.size());
|
|
||||||
|
|
||||||
for (int i = 0; i < ticks.size(); ++i) {
|
|
||||||
runner.TickAt(ticks[i]);
|
|
||||||
runner.ProvideInput(ticks[i], values[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
|
|
||||||
// Make sure we get the right timestamps:
|
|
||||||
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
|
||||||
// Should get the inputs values:
|
|
||||||
EXPECT_THAT(runner.GetOutputValues(), ContainerEq(values));
|
|
||||||
// We should never get the default indication:
|
|
||||||
EXPECT_THAT(runner.GetFlagTimestamps(), IsEmpty());
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
|
|
||||||
// Check that when we provide inputs only part of the time - we get them, but
|
|
||||||
// defaults at the missing times.
|
|
||||||
// That's the usual use case for this calculator.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
const std::vector<int64_t> ticks = {0, 1, 5, 8, 12, 231};
|
|
||||||
// Provide inputs only part of the ticks.
|
|
||||||
// Chosen so there will be defaults before the first input, between the
|
|
||||||
// inputs and after the last input.
|
|
||||||
const std::vector<int64_t> in_ticks = {/*0,*/ 1, 5, /*8,*/ 12, /*, 231*/};
|
|
||||||
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
|
||||||
|
|
||||||
for (int tick : ticks) {
|
|
||||||
runner.TickAt(tick);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < in_ticks.size(); ++i) {
|
|
||||||
runner.ProvideInput(in_ticks[i], in_values[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
|
|
||||||
// Make sure we get all the timestamps:
|
|
||||||
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
|
||||||
// The timestamps of the flag should be exactly the ones not in in_ticks.
|
|
||||||
EXPECT_THAT(runner.GetFlagTimestamps(), ElementsAre(0, 8, 231));
|
|
||||||
// And the values are default in these times, and the input values for
|
|
||||||
// in_ticks.
|
|
||||||
EXPECT_THAT(
|
|
||||||
runner.GetOutputValues(),
|
|
||||||
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
|
|
||||||
// Check that when we provide the inputs not on time - we don't get them.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
|
||||||
// The timestamps chosen so it will be before the first tick, in between ticks
|
|
||||||
// and after the last one. Also - more inputs than ticks.
|
|
||||||
const std::vector<int64_t> in_ticks = {0, 3, 4, 6, 7, 9, 10,
|
|
||||||
11, 13, 14, 15, 16, 232};
|
|
||||||
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
|
||||||
for (int tick : ticks) {
|
|
||||||
runner.TickAt(tick);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < in_ticks.size(); ++i) {
|
|
||||||
runner.ProvideInput(in_ticks[i], in_values[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
|
|
||||||
// Non of the in_ticks should be inserted:
|
|
||||||
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
|
||||||
EXPECT_THAT(runner.GetOutputValues(),
|
|
||||||
AllOf(Each(kDefaultValue), SizeIs(ticks.size())));
|
|
||||||
// All (and only) ticks should get the default.
|
|
||||||
EXPECT_THAT(runner.GetFlagTimestamps(), ContainerEq(ticks));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, FlagValue) {
|
|
||||||
// Since we anyway suppose that the Flag is a bool - there is nothing
|
|
||||||
// interesting to check, but we should check once that the value is the right
|
|
||||||
// (true) one.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
runner.TickAt(0);
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
EXPECT_THAT(runner.GetFlagValues(), ElementsAre(true));
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ValueOrDefaultCalculatorTest, FullTest) {
|
|
||||||
// Make sure that nothing gets wrong with an input that have both right and
|
|
||||||
// wrong timestamps, some defaults etc.
|
|
||||||
ValueOrDefaultRunner runner;
|
|
||||||
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
|
||||||
const std::vector<int64_t> in_ticks = {0, 2, 4, 6, 8, 9, 12, 33, 54, 232};
|
|
||||||
const std::vector<int> in_values = GetIntegersRange(in_ticks.size());
|
|
||||||
|
|
||||||
for (int tick : ticks) {
|
|
||||||
runner.TickAt(tick);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < in_ticks.size(); ++i) {
|
|
||||||
runner.ProvideInput(in_ticks[i], in_values[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_EXPECT_OK(runner.Run());
|
|
||||||
|
|
||||||
EXPECT_THAT(runner.GetOutputTimestamps(), ContainerEq(ticks));
|
|
||||||
// Calculated by hand:
|
|
||||||
EXPECT_THAT(
|
|
||||||
runner.GetOutputValues(),
|
|
||||||
ElementsAre(kDefaultValue, 2, kDefaultValue, 5, 7, 8, kDefaultValue));
|
|
||||||
EXPECT_THAT(runner.GetFlagTimestamps(), ElementsAre(1, 5, 231));
|
|
||||||
EXPECT_THAT(runner.GetFlagValues(), AllOf(Each(true), SizeIs(3)));
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace
|
|
||||||
} // namespace mediapipe
|
|
|
@ -97,7 +97,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:source_location",
|
"//mediapipe/framework/port:source_location",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -126,7 +125,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -153,11 +151,11 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:image_frame_opencv",
|
"//mediapipe/framework/formats:image_frame_opencv",
|
||||||
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:opencv_core",
|
"//mediapipe/framework/port:opencv_core",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
|
@ -204,7 +202,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
|
@ -264,12 +261,9 @@ cc_library(
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/gpu:scale_mode_cc_proto",
|
"//mediapipe/gpu:scale_mode_cc_proto",
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/strings",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"//mediapipe/gpu:gl_base_hdr",
|
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gl_quad_renderer",
|
"//mediapipe/gpu:gl_quad_renderer",
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
"//mediapipe/gpu:gl_simple_shaders",
|
||||||
|
@ -279,38 +273,6 @@ cc_library(
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "image_transformation_calculator_test",
|
|
||||||
srcs = ["image_transformation_calculator_test.cc"],
|
|
||||||
data = ["//mediapipe/calculators/image/testdata:test_images"],
|
|
||||||
tags = [
|
|
||||||
"desktop_only_test",
|
|
||||||
],
|
|
||||||
deps = [
|
|
||||||
":image_transformation_calculator",
|
|
||||||
"//mediapipe/framework:calculator_cc_proto",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework:calculator_runner",
|
|
||||||
"//mediapipe/framework/deps:file_path",
|
|
||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:image_frame",
|
|
||||||
"//mediapipe/framework/formats:image_frame_opencv",
|
|
||||||
"//mediapipe/framework/port:gtest",
|
|
||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
|
||||||
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
|
||||||
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
|
||||||
"//mediapipe/gpu:multi_pool",
|
|
||||||
"//third_party:opencv",
|
|
||||||
"@com_google_absl//absl/container:flat_hash_set",
|
|
||||||
"@com_google_absl//absl/flags:flag",
|
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/strings",
|
|
||||||
"@com_google_googletest//:gtest_main",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "image_cropping_calculator",
|
name = "image_cropping_calculator",
|
||||||
srcs = ["image_cropping_calculator.cc"],
|
srcs = ["image_cropping_calculator.cc"],
|
||||||
|
@ -338,7 +300,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
|
@ -435,7 +396,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:logging",
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -460,8 +420,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/util:image_frame_util",
|
"//mediapipe/util:image_frame_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@libyuv",
|
"@libyuv",
|
||||||
],
|
],
|
||||||
|
@ -667,9 +625,9 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:image",
|
"//mediapipe/framework/formats:image",
|
||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
|
"//mediapipe/framework/port:logging",
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:vector",
|
"//mediapipe/framework/port:vector",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
|
@ -707,7 +665,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:opencv_imgcodecs",
|
"//mediapipe/framework/port:opencv_imgcodecs",
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -730,7 +687,6 @@ cc_library(
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||||
"//mediapipe/gpu:shader_util",
|
"//mediapipe/gpu:shader_util",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
|
@ -788,7 +744,6 @@ cc_library(
|
||||||
":affine_transformation_runner_gl",
|
":affine_transformation_runner_gl",
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/gpu:gpu_service",
|
|
||||||
],
|
],
|
||||||
}) + select({
|
}) + select({
|
||||||
"//mediapipe/framework/port:disable_opencv": [],
|
"//mediapipe/framework/port:disable_opencv": [],
|
||||||
|
|
|
@ -20,7 +20,6 @@
|
||||||
#include "Eigen/Core"
|
#include "Eigen/Core"
|
||||||
#include "Eigen/Geometry"
|
#include "Eigen/Geometry"
|
||||||
#include "Eigen/LU"
|
#include "Eigen/LU"
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
#include "absl/status/status.h"
|
#include "absl/status/status.h"
|
||||||
#include "absl/status/statusor.h"
|
#include "absl/status/statusor.h"
|
||||||
|
@ -54,10 +53,6 @@ bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
|
||||||
#endif // __APPLE__
|
#endif // __APPLE__
|
||||||
case GpuOrigin::TOP_LEFT:
|
case GpuOrigin::TOP_LEFT:
|
||||||
return false;
|
return false;
|
||||||
default:
|
|
||||||
ABSL_LOG(ERROR) << "Incorrect GpuOrigin: "
|
|
||||||
<< static_cast<int>(gpu_origin);
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -223,7 +218,7 @@ class GlTextureWarpAffineRunner
|
||||||
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
|
absl::StrCat(mediapipe::kMediaPipeFragmentShaderPreamble,
|
||||||
interpolation_def, kFragShader);
|
interpolation_def, kFragShader);
|
||||||
|
|
||||||
MP_ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
|
ASSIGN_OR_RETURN(program_, create_fn(vert_src, frag_src));
|
||||||
|
|
||||||
auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
|
auto create_custom_zero_fn = [&]() -> absl::StatusOr<Program> {
|
||||||
std::string custom_zero_border_mode_def = R"(
|
std::string custom_zero_border_mode_def = R"(
|
||||||
|
@ -236,10 +231,10 @@ class GlTextureWarpAffineRunner
|
||||||
};
|
};
|
||||||
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
#if GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||||
if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
|
if (!IsGlClampToBorderSupported(gl_helper_->GetGlContext())) {
|
||||||
MP_ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
MP_ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
ASSIGN_OR_RETURN(program_custom_zero_, create_custom_zero_fn());
|
||||||
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
#endif // GL_CLAMP_TO_BORDER_MAY_BE_SUPPORTED
|
||||||
|
|
||||||
glGenFramebuffers(1, &framebuffer_);
|
glGenFramebuffers(1, &framebuffer_);
|
||||||
|
@ -389,8 +384,6 @@ class GlTextureWarpAffineRunner
|
||||||
glActiveTexture(GL_TEXTURE0);
|
glActiveTexture(GL_TEXTURE0);
|
||||||
glBindTexture(GL_TEXTURE_2D, 0);
|
glBindTexture(GL_TEXTURE_2D, 0);
|
||||||
|
|
||||||
glFlush();
|
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -59,7 +59,7 @@ class OpenCvRunner
|
||||||
const ImageFrame& input, const std::array<float, 16>& matrix,
|
const ImageFrame& input, const std::array<float, 16>& matrix,
|
||||||
const AffineTransformation::Size& size,
|
const AffineTransformation::Size& size,
|
||||||
AffineTransformation::BorderMode border_mode) override {
|
AffineTransformation::BorderMode border_mode) override {
|
||||||
// OpenCV warpAffine works in absolute coordinates, so the transform (which
|
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
|
||||||
// accepts and produces relative coordinates) should be adjusted to first
|
// accepts and produces relative coordinates) should be adjusted to first
|
||||||
// normalize coordinates and then scale them.
|
// normalize coordinates and then scale them.
|
||||||
// clang-format off
|
// clang-format off
|
||||||
|
|
|
@ -15,7 +15,6 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/strings/str_replace.h"
|
#include "absl/strings/str_replace.h"
|
||||||
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
|
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -113,7 +112,7 @@ class BilateralFilterCalculator : public CalculatorBase {
|
||||||
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
REGISTER_CALCULATOR(BilateralFilterCalculator);
|
||||||
|
|
||||||
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
|
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
|
||||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
if (cc->Inputs().HasTag(kInputFrameTag) &&
|
if (cc->Inputs().HasTag(kInputFrameTag) &&
|
||||||
cc->Inputs().HasTag(kInputFrameTagGpu)) {
|
cc->Inputs().HasTag(kInputFrameTagGpu)) {
|
||||||
|
@ -184,8 +183,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
sigma_color_ = options_.sigma_color();
|
sigma_color_ = options_.sigma_color();
|
||||||
sigma_space_ = options_.sigma_space();
|
sigma_space_ = options_.sigma_space();
|
||||||
ABSL_CHECK_GE(sigma_color_, 0.0);
|
CHECK_GE(sigma_color_, 0.0);
|
||||||
ABSL_CHECK_GE(sigma_space_, 0.0);
|
CHECK_GE(sigma_space_, 0.0);
|
||||||
if (!use_gpu_) sigma_color_ *= 255.0;
|
if (!use_gpu_) sigma_color_ *= 255.0;
|
||||||
|
|
||||||
if (use_gpu_) {
|
if (use_gpu_) {
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
@ -26,8 +25,8 @@
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
namespace {
|
namespace {
|
||||||
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
|
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
|
||||||
ABSL_CHECK(mat->depth() == CV_8U);
|
CHECK(mat->depth() == CV_8U);
|
||||||
ABSL_CHECK(channel < mat->channels());
|
CHECK(channel < mat->channels());
|
||||||
const int step = mat->channels();
|
const int step = mat->channels();
|
||||||
for (int r = 0; r < mat->rows; ++r) {
|
for (int r = 0; r < mat->rows; ++r) {
|
||||||
uint8* row_ptr = mat->ptr<uint8>(r);
|
uint8* row_ptr = mat->ptr<uint8>(r);
|
||||||
|
|
|
@ -64,8 +64,7 @@ class ImageCloneCalculator : public Node {
|
||||||
"GPU processing is disabled in build flags");
|
"GPU processing is disabled in build flags");
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||||
cc, /*request_gpu_as_optional=*/true));
|
|
||||||
#endif // MEDIAPIPE_DISABLE_GPU
|
#endif // MEDIAPIPE_DISABLE_GPU
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
@ -73,6 +72,9 @@ class ImageCloneCalculator : public Node {
|
||||||
absl::Status Open(CalculatorContext* cc) override {
|
absl::Status Open(CalculatorContext* cc) override {
|
||||||
const auto& options = cc->Options<mediapipe::ImageCloneCalculatorOptions>();
|
const auto& options = cc->Options<mediapipe::ImageCloneCalculatorOptions>();
|
||||||
output_on_gpu_ = options.output_on_gpu();
|
output_on_gpu_ = options.output_on_gpu();
|
||||||
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
|
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
||||||
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -102,10 +104,6 @@ class ImageCloneCalculator : public Node {
|
||||||
|
|
||||||
if (output_on_gpu_ && !input_on_gpu) {
|
if (output_on_gpu_ && !input_on_gpu) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
if (!gpu_initialized_) {
|
|
||||||
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
|
||||||
gpu_initialized_ = true;
|
|
||||||
}
|
|
||||||
gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); });
|
gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); });
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
} else if (!output_on_gpu_ && input_on_gpu) {
|
} else if (!output_on_gpu_ && input_on_gpu) {
|
||||||
|
@ -120,7 +118,6 @@ class ImageCloneCalculator : public Node {
|
||||||
bool output_on_gpu_;
|
bool output_on_gpu_;
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||||
bool gpu_initialized_ = false;
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
};
|
};
|
||||||
MEDIAPIPE_REGISTER_NODE(ImageCloneCalculator);
|
MEDIAPIPE_REGISTER_NODE(ImageCloneCalculator);
|
||||||
|
|
|
@ -16,7 +16,6 @@
|
||||||
|
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
|
@ -203,9 +202,8 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
|
||||||
|
|
||||||
switch (options.border_mode()) {
|
switch (options.border_mode()) {
|
||||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
|
||||||
<< "BORDER_ZERO mode is not supported by GPU "
|
<< "implementation and will fall back into BORDER_REPLICATE";
|
||||||
<< "implementation and will fall back into BORDER_REPLICATE";
|
|
||||||
break;
|
break;
|
||||||
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
|
||||||
break;
|
break;
|
||||||
|
|
|
@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Output texture buffer dimensions. The values defined in the options will be
|
// Output texture buffer dimensions. The values defined in the options will be
|
||||||
// overridden by the WIDTH and HEIGHT input streams if they exist.
|
// overriden by the WIDTH and HEIGHT input streams if they exist.
|
||||||
optional int32 width = 1;
|
optional int32 width = 1;
|
||||||
optional int32 height = 2;
|
optional int32 height = 2;
|
||||||
|
|
||||||
|
|
|
@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
|
||||||
return focal_length_pixels;
|
return focal_length_pixels;
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::StatusOr<ImageFileProperties> GetImageFileProperties(
|
absl::StatusOr<ImageFileProperties> GetImageFileProperites(
|
||||||
const std::string& image_bytes) {
|
const std::string& image_bytes) {
|
||||||
easyexif::EXIFInfo result;
|
easyexif::EXIFInfo result;
|
||||||
int code = result.parseFrom(image_bytes);
|
int code = result.parseFrom(image_bytes);
|
||||||
|
@ -92,11 +92,11 @@ absl::StatusOr<ImageFileProperties> GetImageFileProperties(
|
||||||
properties.set_focal_length_mm(result.FocalLength);
|
properties.set_focal_length_mm(result.FocalLength);
|
||||||
properties.set_focal_length_35mm(result.FocalLengthIn35mm);
|
properties.set_focal_length_35mm(result.FocalLengthIn35mm);
|
||||||
|
|
||||||
MP_ASSIGN_OR_RETURN(auto focal_length_pixels,
|
ASSIGN_OR_RETURN(auto focal_length_pixels,
|
||||||
ComputeFocalLengthInPixels(properties.image_width(),
|
ComputeFocalLengthInPixels(properties.image_width(),
|
||||||
properties.image_height(),
|
properties.image_height(),
|
||||||
properties.focal_length_35mm(),
|
properties.focal_length_35mm(),
|
||||||
properties.focal_length_mm()));
|
properties.focal_length_mm()));
|
||||||
properties.set_focal_length_pixels(focal_length_pixels);
|
properties.set_focal_length_pixels(focal_length_pixels);
|
||||||
|
|
||||||
return properties;
|
return properties;
|
||||||
|
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
||||||
if (cc->InputSidePackets().NumEntries() == 1) {
|
if (cc->InputSidePackets().NumEntries() == 1) {
|
||||||
const std::string& image_bytes =
|
const std::string& image_bytes =
|
||||||
cc->InputSidePackets().Index(0).Get<std::string>();
|
cc->InputSidePackets().Index(0).Get<std::string>();
|
||||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||||
read_properties_ = true;
|
read_properties_ = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
||||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||||
read_properties_ = true;
|
read_properties_ = true;
|
||||||
}
|
}
|
||||||
if (read_properties_) {
|
if (read_properties_) {
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/status/status.h"
|
|
||||||
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
|
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
|
||||||
#include "mediapipe/calculators/image/rotation_mode.pb.h"
|
#include "mediapipe/calculators/image/rotation_mode.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -28,7 +27,6 @@
|
||||||
#include "mediapipe/gpu/scale_mode.pb.h"
|
#include "mediapipe/gpu/scale_mode.pb.h"
|
||||||
|
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
#include "mediapipe/gpu/gl_base.h"
|
|
||||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||||
#include "mediapipe/gpu/gl_quad_renderer.h"
|
#include "mediapipe/gpu/gl_quad_renderer.h"
|
||||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||||
|
@ -62,42 +60,42 @@ constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
|
||||||
|
|
||||||
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
|
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
|
||||||
switch (rotation) {
|
switch (rotation) {
|
||||||
case mediapipe::RotationMode::UNKNOWN:
|
case mediapipe::RotationMode_Mode_UNKNOWN:
|
||||||
case mediapipe::RotationMode::ROTATION_0:
|
case mediapipe::RotationMode_Mode_ROTATION_0:
|
||||||
return 0;
|
return 0;
|
||||||
case mediapipe::RotationMode::ROTATION_90:
|
case mediapipe::RotationMode_Mode_ROTATION_90:
|
||||||
return 90;
|
return 90;
|
||||||
case mediapipe::RotationMode::ROTATION_180:
|
case mediapipe::RotationMode_Mode_ROTATION_180:
|
||||||
return 180;
|
return 180;
|
||||||
case mediapipe::RotationMode::ROTATION_270:
|
case mediapipe::RotationMode_Mode_ROTATION_270:
|
||||||
return 270;
|
return 270;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
|
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
|
||||||
switch (degrees) {
|
switch (degrees) {
|
||||||
case 0:
|
case 0:
|
||||||
return mediapipe::RotationMode::ROTATION_0;
|
return mediapipe::RotationMode_Mode_ROTATION_0;
|
||||||
case 90:
|
case 90:
|
||||||
return mediapipe::RotationMode::ROTATION_90;
|
return mediapipe::RotationMode_Mode_ROTATION_90;
|
||||||
case 180:
|
case 180:
|
||||||
return mediapipe::RotationMode::ROTATION_180;
|
return mediapipe::RotationMode_Mode_ROTATION_180;
|
||||||
case 270:
|
case 270:
|
||||||
return mediapipe::RotationMode::ROTATION_270;
|
return mediapipe::RotationMode_Mode_ROTATION_270;
|
||||||
default:
|
default:
|
||||||
return mediapipe::RotationMode::UNKNOWN;
|
return mediapipe::RotationMode_Mode_UNKNOWN;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mediapipe::ScaleMode_Mode ParseScaleMode(
|
mediapipe::ScaleMode_Mode ParseScaleMode(
|
||||||
mediapipe::ScaleMode_Mode scale_mode,
|
mediapipe::ScaleMode_Mode scale_mode,
|
||||||
mediapipe::ScaleMode_Mode default_mode) {
|
mediapipe::ScaleMode_Mode default_mode) {
|
||||||
switch (scale_mode) {
|
switch (scale_mode) {
|
||||||
case mediapipe::ScaleMode::DEFAULT:
|
case mediapipe::ScaleMode_Mode_DEFAULT:
|
||||||
return default_mode;
|
return default_mode;
|
||||||
case mediapipe::ScaleMode::STRETCH:
|
case mediapipe::ScaleMode_Mode_STRETCH:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
case mediapipe::ScaleMode::FIT:
|
case mediapipe::ScaleMode_Mode_FIT:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
case mediapipe::ScaleMode::FILL_AND_CROP:
|
case mediapipe::ScaleMode_Mode_FILL_AND_CROP:
|
||||||
return scale_mode;
|
return scale_mode;
|
||||||
default:
|
default:
|
||||||
return default_mode;
|
return default_mode;
|
||||||
|
@ -210,8 +208,6 @@ class ImageTransformationCalculator : public CalculatorBase {
|
||||||
|
|
||||||
bool use_gpu_ = false;
|
bool use_gpu_ = false;
|
||||||
cv::Scalar padding_color_;
|
cv::Scalar padding_color_;
|
||||||
ImageTransformationCalculatorOptions::InterpolationMode interpolation_mode_;
|
|
||||||
|
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
GlCalculatorHelper gpu_helper_;
|
GlCalculatorHelper gpu_helper_;
|
||||||
std::unique_ptr<QuadRenderer> rgb_renderer_;
|
std::unique_ptr<QuadRenderer> rgb_renderer_;
|
||||||
|
@ -347,11 +343,6 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
|
||||||
options_.padding_color().green(),
|
options_.padding_color().green(),
|
||||||
options_.padding_color().blue());
|
options_.padding_color().blue());
|
||||||
|
|
||||||
interpolation_mode_ = options_.interpolation_mode();
|
|
||||||
if (options_.interpolation_mode() ==
|
|
||||||
ImageTransformationCalculatorOptions::DEFAULT) {
|
|
||||||
interpolation_mode_ = ImageTransformationCalculatorOptions::LINEAR;
|
|
||||||
}
|
|
||||||
if (use_gpu_) {
|
if (use_gpu_) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
// Let the helper access the GL context information.
|
// Let the helper access the GL context information.
|
||||||
|
@ -466,48 +457,26 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
||||||
ComputeOutputDimensions(input_width, input_height, &output_width,
|
ComputeOutputDimensions(input_width, input_height, &output_width,
|
||||||
&output_height);
|
&output_height);
|
||||||
|
|
||||||
int opencv_interpolation_mode = cv::INTER_LINEAR;
|
|
||||||
if (output_width_ > 0 && output_height_ > 0) {
|
if (output_width_ > 0 && output_height_ > 0) {
|
||||||
cv::Mat scaled_mat;
|
cv::Mat scaled_mat;
|
||||||
if (scale_mode_ == mediapipe::ScaleMode::STRETCH) {
|
if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
|
||||||
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
|
int scale_flag =
|
||||||
// Use INTER_AREA for downscaling if interpolation mode is set to
|
input_mat.cols > output_width_ && input_mat.rows > output_height_
|
||||||
// LINEAR.
|
? cv::INTER_AREA
|
||||||
if (input_mat.cols > output_width_ && input_mat.rows > output_height_) {
|
: cv::INTER_LINEAR;
|
||||||
opencv_interpolation_mode = cv::INTER_AREA;
|
|
||||||
|
|
||||||
} else {
|
|
||||||
opencv_interpolation_mode = cv::INTER_LINEAR;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
opencv_interpolation_mode = cv::INTER_NEAREST;
|
|
||||||
}
|
|
||||||
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
|
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
|
||||||
0, 0, opencv_interpolation_mode);
|
0, 0, scale_flag);
|
||||||
} else {
|
} else {
|
||||||
const float scale =
|
const float scale =
|
||||||
std::min(static_cast<float>(output_width_) / input_width,
|
std::min(static_cast<float>(output_width_) / input_width,
|
||||||
static_cast<float>(output_height_) / input_height);
|
static_cast<float>(output_height_) / input_height);
|
||||||
const int target_width = std::round(input_width * scale);
|
const int target_width = std::round(input_width * scale);
|
||||||
const int target_height = std::round(input_height * scale);
|
const int target_height = std::round(input_height * scale);
|
||||||
|
int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
|
||||||
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
|
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
|
||||||
// Use INTER_AREA for downscaling if interpolation mode is set to
|
|
||||||
// LINEAR.
|
|
||||||
if (scale < 1.0f) {
|
|
||||||
opencv_interpolation_mode = cv::INTER_AREA;
|
|
||||||
} else {
|
|
||||||
opencv_interpolation_mode = cv::INTER_LINEAR;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
opencv_interpolation_mode = cv::INTER_NEAREST;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
|
||||||
cv::Mat intermediate_mat;
|
cv::Mat intermediate_mat;
|
||||||
cv::resize(input_mat, intermediate_mat,
|
cv::resize(input_mat, intermediate_mat,
|
||||||
cv::Size(target_width, target_height), 0, 0,
|
cv::Size(target_width, target_height), 0, 0, scale_flag);
|
||||||
opencv_interpolation_mode);
|
|
||||||
const int top = (output_height_ - target_height) / 2;
|
const int top = (output_height_ - target_height) / 2;
|
||||||
const int bottom = output_height_ - target_height - top;
|
const int bottom = output_height_ - target_height - top;
|
||||||
const int left = (output_width_ - target_width) / 2;
|
const int left = (output_width_ - target_width) / 2;
|
||||||
|
@ -519,7 +488,7 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
||||||
padding_color_);
|
padding_color_);
|
||||||
} else {
|
} else {
|
||||||
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
|
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
|
||||||
0, 0, opencv_interpolation_mode);
|
0, 0, scale_flag);
|
||||||
output_width = target_width;
|
output_width = target_width;
|
||||||
output_height = target_height;
|
output_height = target_height;
|
||||||
}
|
}
|
||||||
|
@ -545,17 +514,17 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
|
||||||
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
|
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
|
||||||
} else {
|
} else {
|
||||||
switch (rotation_) {
|
switch (rotation_) {
|
||||||
case mediapipe::RotationMode::UNKNOWN:
|
case mediapipe::RotationMode_Mode_UNKNOWN:
|
||||||
case mediapipe::RotationMode::ROTATION_0:
|
case mediapipe::RotationMode_Mode_ROTATION_0:
|
||||||
rotated_mat = input_mat;
|
rotated_mat = input_mat;
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode::ROTATION_90:
|
case mediapipe::RotationMode_Mode_ROTATION_90:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode::ROTATION_180:
|
case mediapipe::RotationMode_Mode_ROTATION_180:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
|
||||||
break;
|
break;
|
||||||
case mediapipe::RotationMode::ROTATION_270:
|
case mediapipe::RotationMode_Mode_ROTATION_270:
|
||||||
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
|
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -592,7 +561,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
||||||
ComputeOutputDimensions(input_width, input_height, &output_width,
|
ComputeOutputDimensions(input_width, input_height, &output_width,
|
||||||
&output_height);
|
&output_height);
|
||||||
|
|
||||||
if (scale_mode_ == mediapipe::ScaleMode::FILL_AND_CROP) {
|
if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
|
||||||
const float scale =
|
const float scale =
|
||||||
std::min(static_cast<float>(output_width_) / input_width,
|
std::min(static_cast<float>(output_width_) / input_width,
|
||||||
static_cast<float>(output_height_) / input_height);
|
static_cast<float>(output_height_) / input_height);
|
||||||
|
@ -656,24 +625,9 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
||||||
input.format());
|
input.format());
|
||||||
|
|
||||||
gpu_helper_.BindFramebuffer(dst);
|
gpu_helper_.BindFramebuffer(dst);
|
||||||
|
|
||||||
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
|
||||||
// In kFit scale mode, the rendered quad does not fill the whole
|
|
||||||
// framebuffer, so clear it beforehand.
|
|
||||||
glClearColor(padding_color_[0] / 255.0f, padding_color_[1] / 255.0f,
|
|
||||||
padding_color_[2] / 255.0f, 1.0f);
|
|
||||||
glClear(GL_COLOR_BUFFER_BIT);
|
|
||||||
}
|
|
||||||
|
|
||||||
glActiveTexture(GL_TEXTURE1);
|
glActiveTexture(GL_TEXTURE1);
|
||||||
glBindTexture(src1.target(), src1.name());
|
glBindTexture(src1.target(), src1.name());
|
||||||
|
|
||||||
if (interpolation_mode_ == ImageTransformationCalculatorOptions::NEAREST) {
|
|
||||||
// TODO: revert texture params.
|
|
||||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
|
|
||||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_RETURN_IF_ERROR(renderer->GlRender(
|
MP_RETURN_IF_ERROR(renderer->GlRender(
|
||||||
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
|
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
|
||||||
rotation, flip_horizontally_, flip_vertically_,
|
rotation, flip_horizontally_, flip_vertically_,
|
||||||
|
@ -698,8 +652,8 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
|
||||||
if (output_width_ > 0 && output_height_ > 0) {
|
if (output_width_ > 0 && output_height_ > 0) {
|
||||||
*output_width = output_width_;
|
*output_width = output_width_;
|
||||||
*output_height = output_height_;
|
*output_height = output_height_;
|
||||||
} else if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
|
} else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
|
||||||
rotation_ == mediapipe::RotationMode::ROTATION_270) {
|
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
|
||||||
*output_width = input_height;
|
*output_width = input_height;
|
||||||
*output_height = input_width;
|
*output_height = input_width;
|
||||||
} else {
|
} else {
|
||||||
|
@ -712,9 +666,9 @@ void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
|
||||||
int input_width, int input_height, int output_width, int output_height,
|
int input_width, int input_height, int output_width, int output_height,
|
||||||
std::array<float, 4>* padding) {
|
std::array<float, 4>* padding) {
|
||||||
padding->fill(0.f);
|
padding->fill(0.f);
|
||||||
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
|
||||||
if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
|
if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
|
||||||
rotation_ == mediapipe::RotationMode::ROTATION_270) {
|
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
|
||||||
std::swap(input_width, input_height);
|
std::swap(input_width, input_height);
|
||||||
}
|
}
|
||||||
const float input_aspect_ratio =
|
const float input_aspect_ratio =
|
||||||
|
|
|
@ -46,24 +46,12 @@ message ImageTransformationCalculatorOptions {
|
||||||
optional bool flip_horizontally = 5 [default = false];
|
optional bool flip_horizontally = 5 [default = false];
|
||||||
// Scale mode.
|
// Scale mode.
|
||||||
optional ScaleMode.Mode scale_mode = 6;
|
optional ScaleMode.Mode scale_mode = 6;
|
||||||
// Padding type. This option is only used when the scale mode is FIT. If set
|
// Padding type. This option is only used when the scale mode is FIT.
|
||||||
// to true (default), a constant border is added with color specified by
|
// Default is to use BORDER_CONSTANT. If set to false, it will use
|
||||||
// padding_color. If set to false, a border is added by replicating edge
|
// BORDER_REPLICATE instead.
|
||||||
// pixels (only supported for CPU).
|
|
||||||
optional bool constant_padding = 7 [default = true];
|
optional bool constant_padding = 7 [default = true];
|
||||||
|
|
||||||
// The color for the padding. This option is only used when the scale mode is
|
// The color for the padding. This option is only used when the scale mode is
|
||||||
// FIT. Default is black.
|
// FIT. Default is black. This is for CPU only.
|
||||||
optional Color padding_color = 8;
|
optional Color padding_color = 8;
|
||||||
|
|
||||||
// Interpolation method to use. Note that on CPU when LINEAR is specified,
|
|
||||||
// INTER_LINEAR is used for upscaling and INTER_AREA is used for downscaling.
|
|
||||||
enum InterpolationMode {
|
|
||||||
DEFAULT = 0;
|
|
||||||
LINEAR = 1;
|
|
||||||
NEAREST = 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mode DEFAULT will use LINEAR interpolation.
|
|
||||||
optional InterpolationMode interpolation_mode = 9;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,430 +0,0 @@
|
||||||
#include <algorithm>
|
|
||||||
#include <string>
|
|
||||||
#include <utility>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "absl/container/flat_hash_set.h"
|
|
||||||
#include "absl/flags/flag.h"
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/strings/substitute.h"
|
|
||||||
#include "mediapipe/framework/calculator.pb.h"
|
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
|
||||||
#include "mediapipe/framework/deps/file_path.h"
|
|
||||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
|
||||||
#include "mediapipe/framework/port/gtest.h"
|
|
||||||
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
|
||||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
|
||||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
|
||||||
#include "mediapipe/gpu/multi_pool.h"
|
|
||||||
#include "testing/base/public/gmock.h"
|
|
||||||
#include "testing/base/public/googletest.h"
|
|
||||||
#include "testing/base/public/gunit.h"
|
|
||||||
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
|
|
||||||
#include "third_party/OpenCV/core/base.hpp"
|
|
||||||
#include "third_party/OpenCV/core/mat.hpp"
|
|
||||||
#include "third_party/OpenCV/core/types.hpp"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
|
||||||
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
absl::flat_hash_set<int> computeUniqueValues(const cv::Mat& mat) {
|
|
||||||
// Compute the unique values in cv::Mat
|
|
||||||
absl::flat_hash_set<int> unique_values;
|
|
||||||
for (int i = 0; i < mat.rows; i++) {
|
|
||||||
for (int j = 0; j < mat.cols; j++) {
|
|
||||||
unique_values.insert(mat.at<unsigned char>(i, j));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return unique_values;
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
|
|
||||||
cv::Mat input_mat;
|
|
||||||
cv::cvtColor(cv::imread(file::JoinPath("./",
|
|
||||||
"/mediapipe/calculators/"
|
|
||||||
"image/testdata/binary_mask.png")),
|
|
||||||
input_mat, cv::COLOR_BGR2GRAY);
|
|
||||||
Packet input_image_packet = MakePacket<ImageFrame>(
|
|
||||||
ImageFormat::GRAY8, input_mat.size().width, input_mat.size().height);
|
|
||||||
input_mat.copyTo(formats::MatView(&(input_image_packet.Get<ImageFrame>())));
|
|
||||||
|
|
||||||
std::vector<std::pair<int, int>> output_dims{
|
|
||||||
{256, 333}, {512, 512}, {1024, 1024}};
|
|
||||||
|
|
||||||
for (auto& output_dim : output_dims) {
|
|
||||||
Packet input_output_dim_packet =
|
|
||||||
MakePacket<std::pair<int, int>>(output_dim);
|
|
||||||
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
|
|
||||||
for (const auto& scale_mode : scale_modes) {
|
|
||||||
CalculatorGraphConfig::Node node_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
|
|
||||||
absl::Substitute(R"(
|
|
||||||
calculator: "ImageTransformationCalculator"
|
|
||||||
input_stream: "IMAGE:input_image"
|
|
||||||
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
|
||||||
output_stream: "IMAGE:output_image"
|
|
||||||
options: {
|
|
||||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
|
||||||
scale_mode: $0
|
|
||||||
interpolation_mode: NEAREST
|
|
||||||
}
|
|
||||||
})",
|
|
||||||
scale_mode));
|
|
||||||
|
|
||||||
CalculatorRunner runner(node_config);
|
|
||||||
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
|
|
||||||
input_image_packet.At(Timestamp(0)));
|
|
||||||
runner.MutableInputs()
|
|
||||||
->Tag("OUTPUT_DIMENSIONS")
|
|
||||||
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(runner.Run());
|
|
||||||
const auto& outputs = runner.Outputs();
|
|
||||||
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
|
|
||||||
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
|
||||||
ABSL_QCHECK_EQ(packets.size(), 1);
|
|
||||||
|
|
||||||
const auto& result = packets[0].Get<ImageFrame>();
|
|
||||||
ASSERT_EQ(output_dim.first, result.Width());
|
|
||||||
ASSERT_EQ(output_dim.second, result.Height());
|
|
||||||
|
|
||||||
auto unique_input_values = computeUniqueValues(input_mat);
|
|
||||||
auto unique_output_values =
|
|
||||||
computeUniqueValues(formats::MatView(&result));
|
|
||||||
EXPECT_THAT(unique_input_values,
|
|
||||||
::testing::ContainerEq(unique_output_values));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ImageTransformationCalculatorTest,
|
|
||||||
NearestNeighborResizingWorksForFloatInput) {
|
|
||||||
cv::Mat input_mat;
|
|
||||||
cv::cvtColor(cv::imread(file::JoinPath("./",
|
|
||||||
"/mediapipe/calculators/"
|
|
||||||
"image/testdata/binary_mask.png")),
|
|
||||||
input_mat, cv::COLOR_BGR2GRAY);
|
|
||||||
Packet input_image_packet = MakePacket<ImageFrame>(
|
|
||||||
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
|
|
||||||
cv::Mat packet_mat_view =
|
|
||||||
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
|
|
||||||
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
|
|
||||||
|
|
||||||
std::vector<std::pair<int, int>> output_dims{
|
|
||||||
{256, 333}, {512, 512}, {1024, 1024}};
|
|
||||||
|
|
||||||
for (auto& output_dim : output_dims) {
|
|
||||||
Packet input_output_dim_packet =
|
|
||||||
MakePacket<std::pair<int, int>>(output_dim);
|
|
||||||
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
|
|
||||||
for (const auto& scale_mode : scale_modes) {
|
|
||||||
CalculatorGraphConfig::Node node_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
|
|
||||||
absl::Substitute(R"(
|
|
||||||
calculator: "ImageTransformationCalculator"
|
|
||||||
input_stream: "IMAGE:input_image"
|
|
||||||
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
|
||||||
output_stream: "IMAGE:output_image"
|
|
||||||
options: {
|
|
||||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
|
||||||
scale_mode: $0
|
|
||||||
interpolation_mode: NEAREST
|
|
||||||
}
|
|
||||||
})",
|
|
||||||
scale_mode));
|
|
||||||
|
|
||||||
CalculatorRunner runner(node_config);
|
|
||||||
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
|
|
||||||
input_image_packet.At(Timestamp(0)));
|
|
||||||
runner.MutableInputs()
|
|
||||||
->Tag("OUTPUT_DIMENSIONS")
|
|
||||||
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(runner.Run());
|
|
||||||
const auto& outputs = runner.Outputs();
|
|
||||||
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
|
|
||||||
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
|
||||||
ABSL_QCHECK_EQ(packets.size(), 1);
|
|
||||||
|
|
||||||
const auto& result = packets[0].Get<ImageFrame>();
|
|
||||||
ASSERT_EQ(output_dim.first, result.Width());
|
|
||||||
ASSERT_EQ(output_dim.second, result.Height());
|
|
||||||
|
|
||||||
auto unique_input_values = computeUniqueValues(packet_mat_view);
|
|
||||||
auto unique_output_values =
|
|
||||||
computeUniqueValues(formats::MatView(&result));
|
|
||||||
EXPECT_THAT(unique_input_values,
|
|
||||||
::testing::ContainerEq(unique_output_values));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
|
|
||||||
cv::Mat input_mat;
|
|
||||||
cv::cvtColor(cv::imread(file::JoinPath("./",
|
|
||||||
"/mediapipe/calculators/"
|
|
||||||
"image/testdata/binary_mask.png")),
|
|
||||||
input_mat, cv::COLOR_BGR2RGBA);
|
|
||||||
|
|
||||||
std::vector<std::pair<int, int>> output_dims{
|
|
||||||
{256, 333}, {512, 512}, {1024, 1024}};
|
|
||||||
|
|
||||||
for (auto& output_dim : output_dims) {
|
|
||||||
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
|
|
||||||
for (const auto& scale_mode : scale_modes) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
absl::Substitute(R"(
|
|
||||||
input_stream: "input_image"
|
|
||||||
input_stream: "image_size"
|
|
||||||
output_stream: "output_image"
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageFrameToGpuBufferCalculator"
|
|
||||||
input_stream: "input_image"
|
|
||||||
output_stream: "input_image_gpu"
|
|
||||||
}
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageTransformationCalculator"
|
|
||||||
input_stream: "IMAGE_GPU:input_image_gpu"
|
|
||||||
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
|
||||||
output_stream: "IMAGE_GPU:output_image_gpu"
|
|
||||||
options: {
|
|
||||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
|
||||||
scale_mode: $0
|
|
||||||
interpolation_mode: NEAREST
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
node {
|
|
||||||
calculator: "GpuBufferToImageFrameCalculator"
|
|
||||||
input_stream: "output_image_gpu"
|
|
||||||
output_stream: "output_image"
|
|
||||||
})",
|
|
||||||
scale_mode));
|
|
||||||
ImageFrame input_image(ImageFormat::SRGBA, input_mat.size().width,
|
|
||||||
input_mat.size().height);
|
|
||||||
input_mat.copyTo(formats::MatView(&input_image));
|
|
||||||
|
|
||||||
std::vector<Packet> output_image_packets;
|
|
||||||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
|
||||||
|
|
||||||
CalculatorGraph graph(graph_config);
|
|
||||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"input_image",
|
|
||||||
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"image_size",
|
|
||||||
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
|
||||||
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
|
|
||||||
|
|
||||||
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
|
||||||
ASSERT_EQ(output_dim.first, output_image.Width());
|
|
||||||
ASSERT_EQ(output_dim.second, output_image.Height());
|
|
||||||
|
|
||||||
auto unique_input_values = computeUniqueValues(input_mat);
|
|
||||||
auto unique_output_values =
|
|
||||||
computeUniqueValues(formats::MatView(&output_image));
|
|
||||||
EXPECT_THAT(unique_input_values,
|
|
||||||
::testing::ContainerEq(unique_output_values));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ImageTransformationCalculatorTest,
|
|
||||||
NearestNeighborResizingWorksForFloatTexture) {
|
|
||||||
cv::Mat input_mat;
|
|
||||||
cv::cvtColor(cv::imread(file::JoinPath("./",
|
|
||||||
"/mediapipe/calculators/"
|
|
||||||
"image/testdata/binary_mask.png")),
|
|
||||||
input_mat, cv::COLOR_BGR2GRAY);
|
|
||||||
Packet input_image_packet = MakePacket<ImageFrame>(
|
|
||||||
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
|
|
||||||
cv::Mat packet_mat_view =
|
|
||||||
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
|
|
||||||
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
|
|
||||||
|
|
||||||
std::vector<std::pair<int, int>> output_dims{
|
|
||||||
{256, 333}, {512, 512}, {1024, 1024}};
|
|
||||||
|
|
||||||
for (auto& output_dim : output_dims) {
|
|
||||||
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
|
|
||||||
for (const auto& scale_mode : scale_modes) {
|
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
|
||||||
absl::Substitute(R"(
|
|
||||||
input_stream: "input_image"
|
|
||||||
input_stream: "image_size"
|
|
||||||
output_stream: "output_image"
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageFrameToGpuBufferCalculator"
|
|
||||||
input_stream: "input_image"
|
|
||||||
output_stream: "input_image_gpu"
|
|
||||||
}
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageTransformationCalculator"
|
|
||||||
input_stream: "IMAGE_GPU:input_image_gpu"
|
|
||||||
input_stream: "OUTPUT_DIMENSIONS:image_size"
|
|
||||||
output_stream: "IMAGE_GPU:output_image_gpu"
|
|
||||||
options: {
|
|
||||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
|
||||||
scale_mode: $0
|
|
||||||
interpolation_mode: NEAREST
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
node {
|
|
||||||
calculator: "GpuBufferToImageFrameCalculator"
|
|
||||||
input_stream: "output_image_gpu"
|
|
||||||
output_stream: "output_image"
|
|
||||||
})",
|
|
||||||
scale_mode));
|
|
||||||
|
|
||||||
std::vector<Packet> output_image_packets;
|
|
||||||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
|
||||||
|
|
||||||
CalculatorGraph graph(graph_config);
|
|
||||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"input_image", input_image_packet.At(Timestamp(0))));
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"image_size",
|
|
||||||
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
|
||||||
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
|
|
||||||
|
|
||||||
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
|
||||||
ASSERT_EQ(output_dim.first, output_image.Width());
|
|
||||||
ASSERT_EQ(output_dim.second, output_image.Height());
|
|
||||||
|
|
||||||
auto unique_input_values = computeUniqueValues(packet_mat_view);
|
|
||||||
auto unique_output_values =
|
|
||||||
computeUniqueValues(formats::MatView(&output_image));
|
|
||||||
EXPECT_THAT(unique_input_values,
|
|
||||||
::testing::ContainerEq(unique_output_values));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST(ImageTransformationCalculatorTest, FitScalingClearsBackground) {
|
|
||||||
// Regression test for not clearing the background in FIT scaling mode.
|
|
||||||
// First scale an all-red (=r) image from 8x4 to 8x4, so it's a plain copy:
|
|
||||||
// rrrrrrrr
|
|
||||||
// rrrrrrrr
|
|
||||||
// rrrrrrrr
|
|
||||||
// rrrrrrrr
|
|
||||||
// Then scale an all-blue image from 4x4 to 8x4 in FIT mode. This should
|
|
||||||
// introduce dark yellow (=y) letterboxes left and right due to padding_color:
|
|
||||||
// yybbbbyy
|
|
||||||
// yybbbbyy
|
|
||||||
// yybbbbyy
|
|
||||||
// yybbbbyy
|
|
||||||
// We make sure that the all-red buffer gets reused. Without clearing the
|
|
||||||
// background, the blue (=b) image will have red letterboxes:
|
|
||||||
// rrbbbbrr
|
|
||||||
// rrbbbbrr
|
|
||||||
// rrbbbbrr
|
|
||||||
// rrbbbbrr
|
|
||||||
|
|
||||||
constexpr int kSmall = 4, kLarge = 8;
|
|
||||||
ImageFrame input_image_red(ImageFormat::SRGBA, kLarge, kSmall);
|
|
||||||
cv::Mat input_image_red_mat = formats::MatView(&input_image_red);
|
|
||||||
input_image_red_mat = cv::Scalar(255, 0, 0, 255);
|
|
||||||
|
|
||||||
ImageFrame input_image_blue(ImageFormat::SRGBA, kSmall, kSmall);
|
|
||||||
cv::Mat input_image_blue_mat = formats::MatView(&input_image_blue);
|
|
||||||
input_image_blue_mat = cv::Scalar(0, 0, 255, 255);
|
|
||||||
|
|
||||||
Packet input_image_red_packet =
|
|
||||||
MakePacket<ImageFrame>(std::move(input_image_red));
|
|
||||||
Packet input_image_blue_packet =
|
|
||||||
MakePacket<ImageFrame>(std::move(input_image_blue));
|
|
||||||
|
|
||||||
CalculatorGraphConfig graph_config =
|
|
||||||
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
|
|
||||||
R"pb(
|
|
||||||
input_stream: "input_image"
|
|
||||||
output_stream: "output_image"
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageFrameToGpuBufferCalculator"
|
|
||||||
input_stream: "input_image"
|
|
||||||
output_stream: "input_image_gpu"
|
|
||||||
}
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "ImageTransformationCalculator"
|
|
||||||
input_stream: "IMAGE_GPU:input_image_gpu"
|
|
||||||
output_stream: "IMAGE_GPU:output_image_gpu"
|
|
||||||
options: {
|
|
||||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
|
||||||
scale_mode: FIT
|
|
||||||
output_width: $0,
|
|
||||||
output_height: $1,
|
|
||||||
padding_color: { red: 128, green: 128, blue: 0 }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
node {
|
|
||||||
calculator: "GpuBufferToImageFrameCalculator"
|
|
||||||
input_stream: "output_image_gpu"
|
|
||||||
output_stream: "output_image"
|
|
||||||
})pb",
|
|
||||||
kLarge, kSmall));
|
|
||||||
|
|
||||||
std::vector<Packet> output_image_packets;
|
|
||||||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
|
||||||
|
|
||||||
CalculatorGraph graph(graph_config);
|
|
||||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
|
||||||
|
|
||||||
// Send the red image multiple times to cause the GPU pool to actually use
|
|
||||||
// a pool.
|
|
||||||
int num_red_packets =
|
|
||||||
std::max(kDefaultMultiPoolOptions.min_requests_before_pool, 1);
|
|
||||||
for (int n = 0; n < num_red_packets; ++n) {
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"input_image", input_image_red_packet.At(Timestamp(n))));
|
|
||||||
}
|
|
||||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
|
||||||
"input_image", input_image_blue_packet.At(Timestamp(num_red_packets))));
|
|
||||||
|
|
||||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
|
||||||
ABSL_QCHECK_EQ(output_image_packets.size(), num_red_packets + 1);
|
|
||||||
|
|
||||||
const auto& output_image_red = output_image_packets[0].Get<ImageFrame>();
|
|
||||||
const auto& output_image_blue =
|
|
||||||
output_image_packets[num_red_packets].Get<ImageFrame>();
|
|
||||||
|
|
||||||
ABSL_QCHECK_EQ(output_image_red.Width(), kLarge);
|
|
||||||
ABSL_QCHECK_EQ(output_image_red.Height(), kSmall);
|
|
||||||
ABSL_QCHECK_EQ(output_image_blue.Width(), kLarge);
|
|
||||||
ABSL_QCHECK_EQ(output_image_blue.Height(), kSmall);
|
|
||||||
|
|
||||||
cv::Mat output_image_blue_mat = formats::MatView(&output_image_blue);
|
|
||||||
ImageFrame expected_image_blue(ImageFormat::SRGBA, kLarge, kSmall);
|
|
||||||
cv::Mat expected_image_blue_mat = formats::MatView(&expected_image_blue);
|
|
||||||
expected_image_blue_mat = cv::Scalar(128, 128, 0, 255);
|
|
||||||
cv::Rect rect((kLarge - kSmall) / 2, 0, kSmall, kSmall);
|
|
||||||
cv::rectangle(expected_image_blue_mat, rect, cv::Scalar(0, 0, 255, 255),
|
|
||||||
cv::FILLED);
|
|
||||||
EXPECT_EQ(cv::sum(cv::sum(output_image_blue_mat != expected_image_blue_mat)),
|
|
||||||
cv::Scalar(0));
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace
|
|
||||||
} // namespace mediapipe
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
|
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
@ -62,7 +61,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
|
||||||
|
|
||||||
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
|
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
|
||||||
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
|
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
|
||||||
ABSL_CHECK_EQ(1, image_frame.ByteDepth());
|
CHECK_EQ(1, image_frame.ByteDepth());
|
||||||
|
|
||||||
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
|
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
|
||||||
absl::make_unique<OpenCvImageEncoderCalculatorResults>();
|
absl::make_unique<OpenCvImageEncoderCalculatorResults>();
|
||||||
|
|
|
@ -18,8 +18,6 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "absl/strings/substitute.h"
|
#include "absl/strings/substitute.h"
|
||||||
#include "libyuv/scale.h"
|
#include "libyuv/scale.h"
|
||||||
|
@ -295,7 +293,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
|
||||||
header->width = output_width_;
|
header->width = output_width_;
|
||||||
header->height = output_height_;
|
header->height = output_height_;
|
||||||
header->format = output_format_;
|
header->format = output_format_;
|
||||||
ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream";
|
LOG(INFO) << "OUTPUTTING HEADER on stream";
|
||||||
cc->Outputs()
|
cc->Outputs()
|
||||||
.Tag("VIDEO_HEADER")
|
.Tag("VIDEO_HEADER")
|
||||||
.Add(header.release(), Timestamp::PreStream());
|
.Add(header.release(), Timestamp::PreStream());
|
||||||
|
@ -395,11 +393,10 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
|
||||||
.SetHeader(Adopt(output_header.release()));
|
.SetHeader(Adopt(output_header.release()));
|
||||||
has_header_ = true;
|
has_header_ = true;
|
||||||
} else {
|
} else {
|
||||||
ABSL_LOG(WARNING)
|
LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient "
|
||||||
<< "Stream had a VideoHeader which didn't have sufficient "
|
"information. "
|
||||||
"information. "
|
"Dropping VideoHeader and trying to deduce needed "
|
||||||
"Dropping VideoHeader and trying to deduce needed "
|
"information.";
|
||||||
"information.";
|
|
||||||
input_width_ = 0;
|
input_width_ = 0;
|
||||||
input_height_ = 0;
|
input_height_ = 0;
|
||||||
if (!options_.has_input_format()) {
|
if (!options_.has_input_format()) {
|
||||||
|
@ -510,7 +507,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame(
|
||||||
|
|
||||||
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
|
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
|
||||||
const YUVImage& yuv_image) {
|
const YUVImage& yuv_image) {
|
||||||
ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
|
CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
|
||||||
if (!has_header_) {
|
if (!has_header_) {
|
||||||
if (input_width_ != yuv_image.width() ||
|
if (input_width_ != yuv_image.width() ||
|
||||||
input_height_ != yuv_image.height()) {
|
input_height_ != yuv_image.height()) {
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/strings/str_split.h"
|
#include "absl/strings/str_split.h"
|
||||||
#include "mediapipe/framework/port/logging.h"
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/ret_check.h"
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
@ -41,10 +40,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
|
||||||
const std::string& max_aspect_ratio, //
|
const std::string& max_aspect_ratio, //
|
||||||
int* crop_width, int* crop_height, //
|
int* crop_width, int* crop_height, //
|
||||||
int* col_start, int* row_start) {
|
int* col_start, int* row_start) {
|
||||||
ABSL_CHECK(crop_width);
|
CHECK(crop_width);
|
||||||
ABSL_CHECK(crop_height);
|
CHECK(crop_height);
|
||||||
ABSL_CHECK(col_start);
|
CHECK(col_start);
|
||||||
ABSL_CHECK(row_start);
|
CHECK(row_start);
|
||||||
|
|
||||||
double min_aspect_ratio_q = 0.0;
|
double min_aspect_ratio_q = 0.0;
|
||||||
double max_aspect_ratio_q = 0.0;
|
double max_aspect_ratio_q = 0.0;
|
||||||
|
@ -84,8 +83,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ABSL_CHECK_LE(*crop_width, input_width);
|
CHECK_LE(*crop_width, input_width);
|
||||||
ABSL_CHECK_LE(*crop_height, input_height);
|
CHECK_LE(*crop_height, input_height);
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,8 +96,8 @@ absl::Status FindOutputDimensions(int input_width, //
|
||||||
bool preserve_aspect_ratio, //
|
bool preserve_aspect_ratio, //
|
||||||
int scale_to_multiple_of, //
|
int scale_to_multiple_of, //
|
||||||
int* output_width, int* output_height) {
|
int* output_width, int* output_height) {
|
||||||
ABSL_CHECK(output_width);
|
CHECK(output_width);
|
||||||
ABSL_CHECK(output_height);
|
CHECK(output_height);
|
||||||
|
|
||||||
if (target_max_area > 0 && input_width * input_height > target_max_area) {
|
if (target_max_area > 0 && input_width * input_height > target_max_area) {
|
||||||
preserve_aspect_ratio = true;
|
preserve_aspect_ratio = true;
|
||||||
|
|
|
@ -15,13 +15,13 @@
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_options.pb.h"
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
#include "mediapipe/framework/formats/image.h"
|
#include "mediapipe/framework/formats/image.h"
|
||||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
#include "mediapipe/framework/port/vector.h"
|
#include "mediapipe/framework/port/vector.h"
|
||||||
|
|
||||||
|
@ -110,15 +110,14 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
|
||||||
|
|
||||||
absl::Status SegmentationSmoothingCalculator::GetContract(
|
absl::Status SegmentationSmoothingCalculator::GetContract(
|
||||||
CalculatorContract* cc) {
|
CalculatorContract* cc) {
|
||||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
|
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
|
||||||
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
|
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
|
||||||
cc->Outputs().Tag(kOutputMaskTag).Set<Image>();
|
cc->Outputs().Tag(kOutputMaskTag).Set<Image>();
|
||||||
|
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||||
cc, /*request_gpu_as_optional=*/true));
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -131,6 +130,10 @@ absl::Status SegmentationSmoothingCalculator::Open(CalculatorContext* cc) {
|
||||||
cc->Options<mediapipe::SegmentationSmoothingCalculatorOptions>();
|
cc->Options<mediapipe::SegmentationSmoothingCalculatorOptions>();
|
||||||
combine_with_previous_ratio_ = options.combine_with_previous_ratio();
|
combine_with_previous_ratio_ = options.combine_with_previous_ratio();
|
||||||
|
|
||||||
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
|
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
||||||
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,9 +154,6 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) {
|
||||||
|
|
||||||
if (use_gpu) {
|
if (use_gpu) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
if (!gpu_initialized_) {
|
|
||||||
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
|
|
||||||
}
|
|
||||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
|
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
|
||||||
if (!gpu_initialized_) {
|
if (!gpu_initialized_) {
|
||||||
MP_RETURN_IF_ERROR(GlSetup(cc));
|
MP_RETURN_IF_ERROR(GlSetup(cc));
|
||||||
|
@ -178,12 +178,10 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) {
|
||||||
|
|
||||||
absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) {
|
absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
if (gpu_initialized_) {
|
gpu_helper_.RunInGlContext([this] {
|
||||||
gpu_helper_.RunInGlContext([this] {
|
if (program_) glDeleteProgram(program_);
|
||||||
if (program_) glDeleteProgram(program_);
|
program_ = 0;
|
||||||
program_ = 0;
|
});
|
||||||
});
|
|
||||||
}
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -275,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) {
|
||||||
|
|
||||||
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
|
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
|
||||||
if (previous_frame.format() != current_frame.format()) {
|
if (previous_frame.format() != current_frame.format()) {
|
||||||
ABSL_LOG(ERROR) << "Warning: mixing input format types. ";
|
LOG(ERROR) << "Warning: mixing input format types. ";
|
||||||
}
|
}
|
||||||
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
|
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
|
||||||
|
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_runner.h"
|
#include "mediapipe/framework/calculator_runner.h"
|
||||||
|
@ -170,7 +169,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ABSL_LOG(ERROR) << "invalid ratio";
|
LOG(ERROR) << "invalid ratio";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -14,13 +14,13 @@
|
||||||
|
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
|
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
#include "mediapipe/framework/calculator_options.pb.h"
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
#include "mediapipe/framework/formats/image_frame.h"
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
#include "mediapipe/framework/port/status.h"
|
#include "mediapipe/framework/port/status.h"
|
||||||
|
@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
|
||||||
REGISTER_CALCULATOR(SetAlphaCalculator);
|
REGISTER_CALCULATOR(SetAlphaCalculator);
|
||||||
|
|
||||||
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
|
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
|
||||||
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
|
CHECK_GE(cc->Inputs().NumEntries(), 1);
|
||||||
|
|
||||||
bool use_gpu = false;
|
bool use_gpu = false;
|
||||||
|
|
||||||
|
@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
|
||||||
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
|
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
|
||||||
const cv::Mat input_mat = formats::MatView(&input_frame);
|
const cv::Mat input_mat = formats::MatView(&input_frame);
|
||||||
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
|
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
|
||||||
ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
|
LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setup destination image
|
// Setup destination image
|
||||||
|
@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
|
||||||
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
|
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
|
||||||
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
|
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
|
||||||
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
|
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
|
||||||
ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported";
|
LOG(ERROR) << "Only RGB or RGBA input image supported";
|
||||||
}
|
}
|
||||||
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
|
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
|
||||||
|
|
||||||
|
|
1
mediapipe/calculators/image/testdata/BUILD
vendored
1
mediapipe/calculators/image/testdata/BUILD
vendored
|
@ -18,7 +18,6 @@ licenses(["notice"])
|
||||||
filegroup(
|
filegroup(
|
||||||
name = "test_images",
|
name = "test_images",
|
||||||
srcs = [
|
srcs = [
|
||||||
"binary_mask.png",
|
|
||||||
"dino.jpg",
|
"dino.jpg",
|
||||||
"dino_quality_50.jpg",
|
"dino_quality_50.jpg",
|
||||||
"dino_quality_80.jpg",
|
"dino_quality_80.jpg",
|
||||||
|
|
BIN
mediapipe/calculators/image/testdata/binary_mask.png
vendored
BIN
mediapipe/calculators/image/testdata/binary_mask.png
vendored
Binary file not shown.
Before Width: | Height: | Size: 771 B |
|
@ -36,7 +36,6 @@
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||||
#include "mediapipe/gpu/gpu_buffer.h"
|
#include "mediapipe/gpu/gpu_buffer.h"
|
||||||
#include "mediapipe/gpu/gpu_service.h"
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
@ -80,8 +79,8 @@ class WarpAffineRunnerHolder<ImageFrame> {
|
||||||
}
|
}
|
||||||
absl::StatusOr<RunnerType*> GetRunner() {
|
absl::StatusOr<RunnerType*> GetRunner() {
|
||||||
if (!runner_) {
|
if (!runner_) {
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(runner_,
|
||||||
runner_, CreateAffineTransformationOpenCvRunner(interpolation_));
|
CreateAffineTransformationOpenCvRunner(interpolation_));
|
||||||
}
|
}
|
||||||
return runner_.get();
|
return runner_.get();
|
||||||
}
|
}
|
||||||
|
@ -107,12 +106,10 @@ class WarpAffineRunnerHolder<mediapipe::GpuBuffer> {
|
||||||
cc->Options<mediapipe::WarpAffineCalculatorOptions>().interpolation());
|
cc->Options<mediapipe::WarpAffineCalculatorOptions>().interpolation());
|
||||||
return gl_helper_->Open(cc);
|
return gl_helper_->Open(cc);
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::StatusOr<RunnerType*> GetRunner() {
|
absl::StatusOr<RunnerType*> GetRunner() {
|
||||||
if (!runner_) {
|
if (!runner_) {
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(runner_, CreateAffineTransformationGlRunner(
|
||||||
runner_, CreateAffineTransformationGlRunner(gl_helper_, gpu_origin_,
|
gl_helper_, gpu_origin_, interpolation_));
|
||||||
interpolation_));
|
|
||||||
}
|
}
|
||||||
return runner_.get();
|
return runner_.get();
|
||||||
}
|
}
|
||||||
|
@ -144,10 +141,7 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
|
||||||
MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
|
MP_RETURN_IF_ERROR(cpu_holder_.Open(cc));
|
||||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
if (cc->Service(kGpuService).IsAvailable()) {
|
MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
|
||||||
MP_RETURN_IF_ERROR(gpu_holder_.Open(cc));
|
|
||||||
gpu_holder_initialized_ = true;
|
|
||||||
}
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
@ -157,28 +151,24 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
|
||||||
AffineTransformation::BorderMode border_mode) override {
|
AffineTransformation::BorderMode border_mode) override {
|
||||||
if (input.UsesGpu()) {
|
if (input.UsesGpu()) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
if (!gpu_holder_initialized_) {
|
ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
|
||||||
return absl::UnavailableError("GPU support is not available");
|
ASSIGN_OR_RETURN(auto result, runner->Run(input.GetGpuBuffer(), matrix,
|
||||||
}
|
size, border_mode));
|
||||||
MP_ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner());
|
|
||||||
MP_ASSIGN_OR_RETURN(
|
|
||||||
auto result,
|
|
||||||
runner->Run(input.GetGpuBuffer(), matrix, size, border_mode));
|
|
||||||
return mediapipe::Image(*result);
|
return mediapipe::Image(*result);
|
||||||
#else
|
#else
|
||||||
return absl::UnavailableError("GPU support is disabled");
|
return absl::UnavailableError("GPU support is disabled");
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
}
|
}
|
||||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||||
MP_ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
|
ASSIGN_OR_RETURN(auto* runner, cpu_holder_.GetRunner());
|
||||||
const auto& frame_ptr = input.GetImageFrameSharedPtr();
|
const auto& frame_ptr = input.GetImageFrameSharedPtr();
|
||||||
// Wrap image into image frame.
|
// Wrap image into image frame.
|
||||||
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
|
const ImageFrame image_frame(frame_ptr->Format(), frame_ptr->Width(),
|
||||||
frame_ptr->Height(), frame_ptr->WidthStep(),
|
frame_ptr->Height(), frame_ptr->WidthStep(),
|
||||||
const_cast<uint8_t*>(frame_ptr->PixelData()),
|
const_cast<uint8_t*>(frame_ptr->PixelData()),
|
||||||
[](uint8_t* data){});
|
[](uint8_t* data){});
|
||||||
MP_ASSIGN_OR_RETURN(auto result,
|
ASSIGN_OR_RETURN(auto result,
|
||||||
runner->Run(image_frame, matrix, size, border_mode));
|
runner->Run(image_frame, matrix, size, border_mode));
|
||||||
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
|
return mediapipe::Image(std::make_shared<ImageFrame>(std::move(result)));
|
||||||
#else
|
#else
|
||||||
return absl::UnavailableError("OpenCV support is disabled");
|
return absl::UnavailableError("OpenCV support is disabled");
|
||||||
|
@ -191,7 +181,6 @@ class WarpAffineRunnerHolder<mediapipe::Image> {
|
||||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
|
WarpAffineRunnerHolder<mediapipe::GpuBuffer> gpu_holder_;
|
||||||
bool gpu_holder_initialized_ = false;
|
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -205,31 +194,27 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
||||||
static absl::Status UpdateContract(CalculatorContract* cc) {
|
static absl::Status UpdateContract(CalculatorContract* cc) {
|
||||||
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
|
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
|
||||||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
|
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
|
||||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
|
||||||
cc, /*request_gpu_as_optional=*/true));
|
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
|
absl::Status Open(CalculatorContext* cc) override { return holder_.Open(cc); }
|
||||||
|
|
||||||
absl::Status Process(CalculatorContext* cc) override {
|
absl::Status Process(CalculatorContext* cc) override {
|
||||||
if (InterfaceT::kInImage(cc).IsEmpty() ||
|
if (InterfaceT::kInImage(cc).IsEmpty() ||
|
||||||
InterfaceT::kMatrix(cc).IsEmpty() ||
|
InterfaceT::kMatrix(cc).IsEmpty() ||
|
||||||
InterfaceT::kOutputSize(cc).IsEmpty()) {
|
InterfaceT::kOutputSize(cc).IsEmpty()) {
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!holder_initialized_) {
|
|
||||||
MP_RETURN_IF_ERROR(holder_.Open(cc));
|
|
||||||
holder_initialized_ = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const std::array<float, 16>& transform = *InterfaceT::kMatrix(cc);
|
const std::array<float, 16>& transform = *InterfaceT::kMatrix(cc);
|
||||||
auto [out_width, out_height] = *InterfaceT::kOutputSize(cc);
|
auto [out_width, out_height] = *InterfaceT::kOutputSize(cc);
|
||||||
AffineTransformation::Size output_size;
|
AffineTransformation::Size output_size;
|
||||||
output_size.width = out_width;
|
output_size.width = out_width;
|
||||||
output_size.height = out_height;
|
output_size.height = out_height;
|
||||||
MP_ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
|
ASSIGN_OR_RETURN(auto* runner, holder_.GetRunner());
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
auto result,
|
auto result,
|
||||||
runner->Run(
|
runner->Run(
|
||||||
*InterfaceT::kInImage(cc), transform, output_size,
|
*InterfaceT::kInImage(cc), transform, output_size,
|
||||||
|
@ -243,7 +228,6 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
||||||
private:
|
private:
|
||||||
WarpAffineRunnerHolder<typename decltype(InterfaceT::kInImage)::PayloadT>
|
WarpAffineRunnerHolder<typename decltype(InterfaceT::kInImage)::PayloadT>
|
||||||
holder_;
|
holder_;
|
||||||
bool holder_initialized_ = false;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
|
@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
|
||||||
.IgnoreError();
|
.IgnoreError();
|
||||||
mediapipe::GetRotatedSubRectToRectTransformMatrix(
|
mediapipe::GetRotatedSubRectToRectTransformMatrix(
|
||||||
roi_absolute, input.cols, input.rows,
|
roi_absolute, input.cols, input.rows,
|
||||||
/*flip_horizontally=*/false, &transform_mat);
|
/*flip_horizontaly=*/false, &transform_mat);
|
||||||
return transform_mat;
|
return transform_mat;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
|
||||||
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
|
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
|
||||||
// YV21) format (as per the `fourcc()` property). This covers the most commonly
|
// YV21) format (as per the `fourcc()` property). This covers the most commonly
|
||||||
// used YUV image formats used on mobile devices. Other formats are not
|
// used YUV image formats used on mobile devices. Other formats are not
|
||||||
// supported and will result in an `InvalidArgumentError`.
|
// supported and wil result in an `InvalidArgumentError`.
|
||||||
class YUVToImageCalculator : public Node {
|
class YUVToImageCalculator : public Node {
|
||||||
public:
|
public:
|
||||||
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
|
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
|
||||||
|
|
|
@ -31,14 +31,12 @@ mediapipe_proto_library(
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "callback_packet_calculator",
|
name = "callback_packet_calculator",
|
||||||
srcs = ["callback_packet_calculator.cc"],
|
srcs = ["callback_packet_calculator.cc"],
|
||||||
hdrs = ["callback_packet_calculator.h"],
|
|
||||||
visibility = ["//mediapipe/framework:__subpackages__"],
|
visibility = ["//mediapipe/framework:__subpackages__"],
|
||||||
deps = [
|
deps = [
|
||||||
":callback_packet_calculator_cc_proto",
|
":callback_packet_calculator_cc_proto",
|
||||||
"//mediapipe/framework:calculator_base",
|
"//mediapipe/framework:calculator_base",
|
||||||
"//mediapipe/framework:calculator_registry",
|
"//mediapipe/framework:calculator_registry",
|
||||||
"//mediapipe/framework:output_side_packet",
|
"//mediapipe/framework:output_side_packet",
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
|
@ -11,12 +11,10 @@
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
#include "mediapipe/calculators/internal/callback_packet_calculator.h"
|
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
#include "absl/status/status.h"
|
|
||||||
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
|
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
|
||||||
#include "mediapipe/framework/calculator_base.h"
|
#include "mediapipe/framework/calculator_base.h"
|
||||||
#include "mediapipe/framework/calculator_registry.h"
|
#include "mediapipe/framework/calculator_registry.h"
|
||||||
|
@ -41,55 +39,64 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
|
||||||
*post_stream_packet = packet;
|
*post_stream_packet = packet;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) {
|
// Creates a callback which takes a packet and stores it either in a
|
||||||
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
// vector of packets or stores only the packet at PostStream timestamp.
|
||||||
switch (options.type()) {
|
// The kind of callback is controlled by an option. The callback is
|
||||||
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
// a std::function and is directly usable by CallbackCalculator.
|
||||||
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
// Since the options for the packet generator include a serialized pointer
|
||||||
cc->OutputSidePackets()
|
// value, the resulting callback is only valid on the original machine
|
||||||
.Index(0)
|
// while that pointer is still alive.
|
||||||
.Set<std::function<void(const Packet&)>>();
|
class CallbackPacketCalculator : public CalculatorBase {
|
||||||
break;
|
public:
|
||||||
default:
|
static absl::Status GetContract(CalculatorContract* cc) {
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
||||||
<< "Invalid type of callback to produce.";
|
switch (options.type()) {
|
||||||
|
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
||||||
|
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
||||||
|
cc->OutputSidePackets()
|
||||||
|
.Index(0)
|
||||||
|
.Set<std::function<void(const Packet&)>>();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
|
<< "Invalid type of callback to produce.";
|
||||||
|
}
|
||||||
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) {
|
absl::Status Open(CalculatorContext* cc) override {
|
||||||
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
|
||||||
void* ptr;
|
void* ptr;
|
||||||
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
|
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
|
||||||
<< "Stored pointer value in options is invalid.";
|
|
||||||
}
|
|
||||||
switch (options.type()) {
|
|
||||||
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
|
||||||
cc->OutputSidePackets().Index(0).Set(
|
|
||||||
MakePacket<std::function<void(const Packet&)>>(std::bind(
|
|
||||||
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
|
|
||||||
std::placeholders::_1)));
|
|
||||||
break;
|
|
||||||
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
|
||||||
cc->OutputSidePackets().Index(0).Set(
|
|
||||||
MakePacket<std::function<void(const Packet&)>>(
|
|
||||||
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
|
|
||||||
std::placeholders::_1)));
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
<< "Invalid type to dump into.";
|
<< "Stored pointer value in options is invalid.";
|
||||||
|
}
|
||||||
|
switch (options.type()) {
|
||||||
|
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
|
||||||
|
cc->OutputSidePackets().Index(0).Set(
|
||||||
|
MakePacket<std::function<void(const Packet&)>>(std::bind(
|
||||||
|
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
|
||||||
|
std::placeholders::_1)));
|
||||||
|
break;
|
||||||
|
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
|
||||||
|
cc->OutputSidePackets().Index(0).Set(
|
||||||
|
MakePacket<std::function<void(const Packet&)>>(
|
||||||
|
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
|
||||||
|
std::placeholders::_1)));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
|
||||||
|
<< "Invalid type to dump into.";
|
||||||
|
}
|
||||||
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) {
|
absl::Status Process(CalculatorContext* cc) override {
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
REGISTER_CALCULATOR(CallbackPacketCalculator);
|
REGISTER_CALCULATOR(CallbackPacketCalculator);
|
||||||
|
|
||||||
|
|
|
@ -1,39 +0,0 @@
|
||||||
// Copyright 2023 The MediaPipe Authors.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
|
||||||
#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
|
||||||
|
|
||||||
#include "absl/status/status.h"
|
|
||||||
#include "mediapipe/framework/calculator_base.h"
|
|
||||||
|
|
||||||
namespace mediapipe {
|
|
||||||
|
|
||||||
// Creates a callback which takes a packet and stores it either in a
|
|
||||||
// vector of packets or stores only the packet at PostStream timestamp.
|
|
||||||
// The kind of callback is controlled by an option. The callback is
|
|
||||||
// a std::function and is directly usable by CallbackCalculator.
|
|
||||||
// Since the options for the packet generator include a serialized pointer
|
|
||||||
// value, the resulting callback is only valid on the original machine
|
|
||||||
// while that pointer is still alive.
|
|
||||||
class CallbackPacketCalculator : public CalculatorBase {
|
|
||||||
public:
|
|
||||||
static absl::Status GetContract(CalculatorContract* cc);
|
|
||||||
absl::Status Open(CalculatorContext* cc) override;
|
|
||||||
absl::Status Process(CalculatorContext* cc) override;
|
|
||||||
};
|
|
||||||
|
|
||||||
} // namespace mediapipe
|
|
||||||
|
|
||||||
#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
|
|
|
@ -13,16 +13,16 @@
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
#
|
#
|
||||||
|
|
||||||
|
load("@bazel_skylib//lib:selects.bzl", "selects")
|
||||||
|
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||||
load(
|
load(
|
||||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||||
"mediapipe_binary_graph",
|
"mediapipe_binary_graph",
|
||||||
)
|
)
|
||||||
load("@bazel_skylib//lib:selects.bzl", "selects")
|
|
||||||
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
|
|
||||||
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
|
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
|
||||||
load("//mediapipe/framework:more_selects.bzl", "more_selects")
|
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
|
||||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
|
||||||
load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite")
|
load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite")
|
||||||
|
load("//mediapipe/framework:more_selects.bzl", "more_selects")
|
||||||
|
|
||||||
licenses(["notice"])
|
licenses(["notice"])
|
||||||
|
|
||||||
|
@ -50,18 +50,10 @@ more_selects.config_setting_negation(
|
||||||
)
|
)
|
||||||
|
|
||||||
selects.config_setting_group(
|
selects.config_setting_group(
|
||||||
name = "platform_apple_with_gpu",
|
name = "platform_ios_with_gpu",
|
||||||
match_all = [
|
match_all = [
|
||||||
":not_disable_gpu",
|
":not_disable_gpu",
|
||||||
"//mediapipe:apple",
|
"//mediapipe:ios",
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
selects.config_setting_group(
|
|
||||||
name = "platform_apple_without_gpu",
|
|
||||||
match_all = [
|
|
||||||
":disable_gpu",
|
|
||||||
"//mediapipe:apple",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -95,7 +87,6 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
"//mediapipe/framework/formats:time_series_header_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/util:time_series_util",
|
"//mediapipe/util:time_series_util",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
|
@ -190,7 +181,6 @@ cc_library(
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework/api2:node",
|
"//mediapipe/framework/api2:node",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
|
@ -208,7 +198,6 @@ cc_test(
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@org_tensorflow//tensorflow/lite/c:common",
|
"@org_tensorflow//tensorflow/lite/c:common",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -262,7 +251,7 @@ cc_test(
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_sentencepiece//:sentencepiece_processor", # fixdeps: keep
|
"@com_google_sentencepiece//src:sentencepiece_processor", # fixdeps: keep
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -311,7 +300,7 @@ cc_test(
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_sentencepiece//:sentencepiece_processor", # fixdeps: keep
|
"@com_google_sentencepiece//src:sentencepiece_processor", # fixdeps: keep
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -456,7 +445,6 @@ cc_library(
|
||||||
"//mediapipe/framework/deps:file_path",
|
"//mediapipe/framework/deps:file_path",
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/util/tflite:tflite_gpu_runner",
|
"//mediapipe/util/tflite:tflite_gpu_runner",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/status",
|
"@com_google_absl//absl/status",
|
||||||
"@com_google_absl//absl/status:statusor",
|
"@com_google_absl//absl/status:statusor",
|
||||||
|
@ -486,7 +474,6 @@ cc_library(
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/objc:mediapipe_framework_ios",
|
"//mediapipe/objc:mediapipe_framework_ios",
|
||||||
"//mediapipe/util/tflite:config",
|
"//mediapipe/util/tflite:config",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/strings:str_format",
|
"@com_google_absl//absl/strings:str_format",
|
||||||
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
|
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
|
||||||
|
@ -622,7 +609,7 @@ cc_library(
|
||||||
":inference_calculator_interface",
|
":inference_calculator_interface",
|
||||||
] + select({
|
] + select({
|
||||||
"//conditions:default": [":inference_calculator_gl_if_compute_shader_available"],
|
"//conditions:default": [":inference_calculator_gl_if_compute_shader_available"],
|
||||||
":platform_apple_with_gpu": [":inference_calculator_metal"],
|
":platform_ios_with_gpu": [":inference_calculator_metal"],
|
||||||
}),
|
}),
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -633,7 +620,6 @@ mediapipe_proto_library(
|
||||||
deps = [
|
deps = [
|
||||||
"//mediapipe/framework:calculator_options_proto",
|
"//mediapipe/framework:calculator_options_proto",
|
||||||
"//mediapipe/framework:calculator_proto",
|
"//mediapipe/framework:calculator_proto",
|
||||||
"//mediapipe/gpu:gpu_origin_proto",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -657,36 +643,16 @@ cc_library(
|
||||||
}),
|
}),
|
||||||
deps = [
|
deps = [
|
||||||
":tensor_converter_calculator_cc_proto",
|
":tensor_converter_calculator_cc_proto",
|
||||||
":tensor_converter_cpu",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework:port",
|
"//mediapipe/framework:port",
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:matrix",
|
"//mediapipe/framework/formats:matrix",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
"//mediapipe/framework/port:statusor",
|
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
|
||||||
"//mediapipe/gpu:gpu_buffer_format",
|
|
||||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
|
||||||
"//mediapipe/util:resource_util",
|
"//mediapipe/util:resource_util",
|
||||||
"@com_google_absl//absl/log",
|
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/log:check",
|
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
"@com_google_absl//absl/strings",
|
|
||||||
"@com_google_absl//absl/strings:str_format",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||||
"tensor_converter_calculator_gpu_deps",
|
|
||||||
"//mediapipe/gpu:gl_base",
|
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
|
||||||
"//mediapipe/gpu:shader_util",
|
|
||||||
],
|
|
||||||
}) + select({
|
}) + select({
|
||||||
"//mediapipe:apple": [
|
"//mediapipe:apple": [
|
||||||
"//third_party/apple_frameworks:MetalKit",
|
"//third_party/apple_frameworks:MetalKit",
|
||||||
|
@ -696,35 +662,6 @@ cc_library(
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensor_converter_cpu",
|
|
||||||
srcs = ["tensor_converter_cpu.cc"],
|
|
||||||
hdrs = ["tensor_converter_cpu.h"],
|
|
||||||
deps = [
|
|
||||||
"//mediapipe/framework/formats:image_frame",
|
|
||||||
"//mediapipe/framework/formats:matrix",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "tensor_converter_cpu_test",
|
|
||||||
srcs = ["tensor_converter_cpu_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":tensor_converter_cpu",
|
|
||||||
"//mediapipe/framework/formats:matrix",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:gtest",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
"//mediapipe/framework/port:status_matchers",
|
|
||||||
"//mediapipe/util:image_test_utils",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "tensor_converter_calculator_gpu_deps",
|
name = "tensor_converter_calculator_gpu_deps",
|
||||||
visibility = ["//visibility:private"],
|
visibility = ["//visibility:private"],
|
||||||
|
@ -733,13 +670,12 @@ cc_library(
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
],
|
],
|
||||||
":platform_apple_with_gpu": [
|
"//mediapipe:ios": [
|
||||||
"//mediapipe/gpu:MPPMetalHelper",
|
"//mediapipe/gpu:MPPMetalHelper",
|
||||||
"//mediapipe/gpu:MPPMetalUtil",
|
"//mediapipe/gpu:MPPMetalUtil",
|
||||||
"//mediapipe/objc:mediapipe_framework_ios",
|
"//mediapipe/objc:mediapipe_framework_ios",
|
||||||
],
|
],
|
||||||
# This setting is needed to allow bazel to build all targets on Mac with GPU disabled
|
"//mediapipe:macos": [],
|
||||||
":platform_apple_without_gpu": [],
|
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
"//mediapipe/gpu:gl_simple_shaders",
|
||||||
|
@ -763,11 +699,9 @@ cc_test(
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:gtest_main",
|
"//mediapipe/framework/port:gtest_main",
|
||||||
"//mediapipe/framework/port:integral_types",
|
"//mediapipe/framework/port:integral_types",
|
||||||
"//mediapipe/framework/port:opencv_core",
|
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"//mediapipe/framework/tool:validate_type",
|
"//mediapipe/framework/tool:validate_type",
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -803,8 +737,6 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
|
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings:str_format",
|
"@com_google_absl//absl/strings:str_format",
|
||||||
"@com_google_absl//absl/types:span",
|
"@com_google_absl//absl/types:span",
|
||||||
] + selects.with_or({
|
] + selects.with_or({
|
||||||
|
@ -824,12 +756,11 @@ cc_library(
|
||||||
name = "tensors_to_detections_calculator_gpu_deps",
|
name = "tensors_to_detections_calculator_gpu_deps",
|
||||||
visibility = ["//visibility:private"],
|
visibility = ["//visibility:private"],
|
||||||
deps = select({
|
deps = select({
|
||||||
":platform_apple_with_gpu": [
|
"//mediapipe:ios": [
|
||||||
"//mediapipe/gpu:MPPMetalHelper",
|
"//mediapipe/gpu:MPPMetalHelper",
|
||||||
"//mediapipe/gpu:MPPMetalUtil",
|
"//mediapipe/gpu:MPPMetalUtil",
|
||||||
],
|
],
|
||||||
# This setting is needed to allow bazel to build all targets on Mac with GPU disabled
|
"//mediapipe:macos": [],
|
||||||
":platform_apple_without_gpu": [],
|
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
],
|
],
|
||||||
|
@ -862,7 +793,6 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
],
|
],
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
@ -1028,48 +958,6 @@ cc_test(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensor_to_joints_calculator",
|
|
||||||
srcs = ["tensor_to_joints_calculator.cc"],
|
|
||||||
hdrs = ["tensor_to_joints_calculator.h"],
|
|
||||||
deps = [
|
|
||||||
":tensor_to_joints_calculator_cc_proto",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework/api2:node",
|
|
||||||
"//mediapipe/framework/formats:body_rig_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
],
|
|
||||||
alwayslink = 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
mediapipe_proto_library(
|
|
||||||
name = "tensor_to_joints_calculator_proto",
|
|
||||||
srcs = ["tensor_to_joints_calculator.proto"],
|
|
||||||
deps = [
|
|
||||||
"//mediapipe/framework:calculator_options_proto",
|
|
||||||
"//mediapipe/framework:calculator_proto",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "tensor_to_joints_calculator_test",
|
|
||||||
srcs = ["tensor_to_joints_calculator_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":tensor_to_joints_calculator",
|
|
||||||
":tensor_to_joints_calculator_cc_proto",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework:calculator_runner",
|
|
||||||
"//mediapipe/framework:timestamp",
|
|
||||||
"//mediapipe/framework/formats:body_rig_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
|
||||||
"@com_google_absl//absl/strings",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "image_to_tensor_calculator",
|
name = "image_to_tensor_calculator",
|
||||||
srcs = ["image_to_tensor_calculator.cc"],
|
srcs = ["image_to_tensor_calculator.cc"],
|
||||||
|
@ -1097,8 +985,6 @@ cc_library(
|
||||||
"//mediapipe/framework/port:status",
|
"//mediapipe/framework/port:status",
|
||||||
"//mediapipe/framework/port:statusor",
|
"//mediapipe/framework/port:statusor",
|
||||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
] + select({
|
] + select({
|
||||||
"//mediapipe/gpu:disable_gpu": [],
|
"//mediapipe/gpu:disable_gpu": [],
|
||||||
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
|
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
|
||||||
|
@ -1191,7 +1077,6 @@ cc_test(
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
"//mediapipe/framework/port:parse_text_proto",
|
||||||
"//mediapipe/util:image_test_utils",
|
"//mediapipe/util:image_test_utils",
|
||||||
"@com_google_absl//absl/flags:flag",
|
"@com_google_absl//absl/flags:flag",
|
||||||
"@com_google_absl//absl/log:absl_check",
|
|
||||||
"@com_google_absl//absl/memory",
|
"@com_google_absl//absl/memory",
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_absl//absl/strings:str_format",
|
"@com_google_absl//absl/strings:str_format",
|
||||||
|
@ -1319,7 +1204,6 @@ cc_library(
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
"//mediapipe/gpu:gl_simple_shaders",
|
||||||
"//mediapipe/gpu:shader_util",
|
"//mediapipe/gpu:shader_util",
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
|
@ -1452,8 +1336,6 @@ cc_library(
|
||||||
}),
|
}),
|
||||||
deps = [
|
deps = [
|
||||||
":tensors_to_segmentation_calculator_cc_proto",
|
":tensors_to_segmentation_calculator_cc_proto",
|
||||||
":tensors_to_segmentation_converter",
|
|
||||||
":tensors_to_segmentation_utils",
|
|
||||||
"//mediapipe/framework:calculator_context",
|
"//mediapipe/framework:calculator_context",
|
||||||
"//mediapipe/framework:calculator_framework",
|
"//mediapipe/framework:calculator_framework",
|
||||||
"//mediapipe/framework:port",
|
"//mediapipe/framework:port",
|
||||||
|
@ -1461,11 +1343,9 @@ cc_library(
|
||||||
"//mediapipe/framework/formats:image_frame",
|
"//mediapipe/framework/formats:image_frame",
|
||||||
"//mediapipe/framework/formats:tensor",
|
"//mediapipe/framework/formats:tensor",
|
||||||
"//mediapipe/framework/port:ret_check",
|
"//mediapipe/framework/port:ret_check",
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
"//mediapipe/framework/port:statusor",
|
"//mediapipe/framework/port:statusor",
|
||||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||||
"//mediapipe/util:resource_util",
|
"//mediapipe/util:resource_util",
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/strings",
|
"@com_google_absl//absl/strings",
|
||||||
"@com_google_absl//absl/strings:str_format",
|
"@com_google_absl//absl/strings:str_format",
|
||||||
"@com_google_absl//absl/types:span",
|
"@com_google_absl//absl/types:span",
|
||||||
|
@ -1476,12 +1356,11 @@ cc_library(
|
||||||
"//mediapipe/gpu:gl_calculator_helper",
|
"//mediapipe/gpu:gl_calculator_helper",
|
||||||
"//mediapipe/gpu:gl_simple_shaders",
|
"//mediapipe/gpu:gl_simple_shaders",
|
||||||
"//mediapipe/gpu:gpu_buffer",
|
"//mediapipe/gpu:gpu_buffer",
|
||||||
"//mediapipe/gpu:gpu_buffer_format",
|
|
||||||
"//mediapipe/gpu:shader_util",
|
"//mediapipe/gpu:shader_util",
|
||||||
],
|
],
|
||||||
}) + selects.with_or({
|
}) + selects.with_or({
|
||||||
":gpu_inference_disabled": [],
|
":gpu_inference_disabled": [],
|
||||||
":platform_apple_with_gpu": [
|
"//mediapipe:ios": [
|
||||||
"//mediapipe/gpu:MPPMetalUtil",
|
"//mediapipe/gpu:MPPMetalUtil",
|
||||||
"//mediapipe/gpu:MPPMetalHelper",
|
"//mediapipe/gpu:MPPMetalHelper",
|
||||||
"//third_party/apple_frameworks:MetalKit",
|
"//third_party/apple_frameworks:MetalKit",
|
||||||
|
@ -1496,109 +1375,13 @@ cc_library(
|
||||||
}) + select({
|
}) + select({
|
||||||
"//mediapipe/framework/port:disable_opencv": [],
|
"//mediapipe/framework/port:disable_opencv": [],
|
||||||
"//conditions:default": [
|
"//conditions:default": [
|
||||||
":tensors_to_segmentation_converter_opencv",
|
"//mediapipe/framework/formats:image_opencv",
|
||||||
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
alwayslink = 1,
|
alwayslink = 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensors_to_segmentation_utils",
|
|
||||||
srcs = ["tensors_to_segmentation_utils.cc"],
|
|
||||||
hdrs = ["tensors_to_segmentation_utils.h"],
|
|
||||||
deps = [
|
|
||||||
"//mediapipe/framework:port",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "tensors_to_segmentation_utils_test",
|
|
||||||
srcs = ["tensors_to_segmentation_utils_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":tensors_to_segmentation_utils",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
"//mediapipe/framework/port:status_matchers",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensors_to_segmentation_converter",
|
|
||||||
hdrs = ["tensors_to_segmentation_converter.h"],
|
|
||||||
deps = [
|
|
||||||
"//mediapipe/framework/formats:image",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensors_to_segmentation_converter_opencv",
|
|
||||||
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
|
|
||||||
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
|
|
||||||
deps = [
|
|
||||||
":tensors_to_segmentation_calculator_cc_proto",
|
|
||||||
":tensors_to_segmentation_converter",
|
|
||||||
":tensors_to_segmentation_utils",
|
|
||||||
"//mediapipe/framework/formats:image",
|
|
||||||
"//mediapipe/framework/formats:image_frame",
|
|
||||||
"//mediapipe/framework/formats:image_opencv",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:opencv_core",
|
|
||||||
"//mediapipe/framework/port:opencv_imgproc",
|
|
||||||
"//mediapipe/framework/port:ret_check",
|
|
||||||
"//mediapipe/framework/port:status",
|
|
||||||
"@com_google_absl//absl/status",
|
|
||||||
"@com_google_absl//absl/status:statusor",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
|
||||||
name = "tensors_to_segmentation_calculator_test_utils",
|
|
||||||
testonly = 1,
|
|
||||||
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
|
|
||||||
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
|
|
||||||
deps = [
|
|
||||||
":tensors_to_segmentation_calculator_cc_proto",
|
|
||||||
"//mediapipe/framework:calculator_cc_proto",
|
|
||||||
"//mediapipe/framework/port:parse_text_proto",
|
|
||||||
"@com_google_absl//absl/log:absl_log",
|
|
||||||
"@com_google_absl//absl/strings",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "tensors_to_segmentation_calculator_test_utils_test",
|
|
||||||
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":tensors_to_segmentation_calculator_cc_proto",
|
|
||||||
":tensors_to_segmentation_calculator_test_utils",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_test(
|
|
||||||
name = "tensors_to_segmentation_calculator_test",
|
|
||||||
srcs = ["tensors_to_segmentation_calculator_test.cc"],
|
|
||||||
deps = [
|
|
||||||
":tensors_to_segmentation_calculator",
|
|
||||||
":tensors_to_segmentation_calculator_cc_proto",
|
|
||||||
":tensors_to_segmentation_calculator_test_utils",
|
|
||||||
"//mediapipe/framework:calculator_framework",
|
|
||||||
"//mediapipe/framework:calculator_runner",
|
|
||||||
"//mediapipe/framework:packet",
|
|
||||||
"//mediapipe/framework:timestamp",
|
|
||||||
"//mediapipe/framework/formats:image",
|
|
||||||
"//mediapipe/framework/formats:image_format_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:image_opencv",
|
|
||||||
"//mediapipe/framework/formats:rect_cc_proto",
|
|
||||||
"//mediapipe/framework/formats:tensor",
|
|
||||||
"//mediapipe/framework/port:gtest_main",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "tensors_dequantization_calculator",
|
name = "tensors_dequantization_calculator",
|
||||||
srcs = ["tensors_dequantization_calculator.cc"],
|
srcs = ["tensors_dequantization_calculator.cc"],
|
||||||
|
|
|
@ -20,7 +20,6 @@
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
#include "absl/status/status.h"
|
#include "absl/status/status.h"
|
||||||
#include "absl/status/statusor.h"
|
#include "absl/status/statusor.h"
|
||||||
|
@ -109,7 +108,7 @@ bool IsValidFftSize(int size) {
|
||||||
// Non-streaming mode: when "stream_mode" is set to false in the calculator
|
// Non-streaming mode: when "stream_mode" is set to false in the calculator
|
||||||
// options, the calculators treats the packets in the input audio stream as
|
// options, the calculators treats the packets in the input audio stream as
|
||||||
// a batch of unrelated audio buffers. In each Process() call, the input
|
// a batch of unrelated audio buffers. In each Process() call, the input
|
||||||
// buffer will be first resampled, and framed as fixed-sized, possibly
|
// buffer will be frist resampled, and framed as fixed-sized, possibly
|
||||||
// overlapping tensors. The last tensor produced by a Process() invocation
|
// overlapping tensors. The last tensor produced by a Process() invocation
|
||||||
// will be zero-padding if the remaining samples are insufficient. As the
|
// will be zero-padding if the remaining samples are insufficient. As the
|
||||||
// calculator treats the input packets as unrelated, all samples will be
|
// calculator treats the input packets as unrelated, all samples will be
|
||||||
|
@ -159,7 +158,7 @@ class AudioToTensorCalculator : public Node {
|
||||||
public:
|
public:
|
||||||
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
|
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
|
||||||
// TODO: Removes this optional input stream when the "AUDIO" stream
|
// TODO: Removes this optional input stream when the "AUDIO" stream
|
||||||
// uses the new mediapipe audio data containers that carry audio metadata,
|
// uses the new mediapipe audio data containers that carry audio metatdata,
|
||||||
// such as sample rate.
|
// such as sample rate.
|
||||||
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
|
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
|
||||||
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
|
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
|
||||||
|
@ -349,7 +348,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
|
||||||
return absl::InvalidArgumentError(
|
return absl::InvalidArgumentError(
|
||||||
"The audio data should be stored in column-major.");
|
"The audio data should be stored in column-major.");
|
||||||
}
|
}
|
||||||
ABSL_CHECK(channels_match || mono_output);
|
CHECK(channels_match || mono_output);
|
||||||
const Matrix& input = channels_match ? input_frame
|
const Matrix& input = channels_match ? input_frame
|
||||||
// Mono mixdown.
|
// Mono mixdown.
|
||||||
: input_frame.colwise().mean();
|
: input_frame.colwise().mean();
|
||||||
|
@ -458,7 +457,7 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
|
||||||
}
|
}
|
||||||
|
|
||||||
void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
|
void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
|
||||||
ABSL_CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
|
CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
|
||||||
if (num_samples == 0) {
|
if (num_samples == 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -517,8 +516,8 @@ absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
|
||||||
// The last two elements are Nyquist component.
|
// The last two elements are Nyquist component.
|
||||||
fft_output_matrix(fft_size_ - 2) = fft_output_[1]; // Nyquist real part
|
fft_output_matrix(fft_size_ - 2) = fft_output_[1]; // Nyquist real part
|
||||||
fft_output_matrix(fft_size_ - 1) = 0.0f; // Nyquist imagery part
|
fft_output_matrix(fft_size_ - 1) = 0.0f; // Nyquist imagery part
|
||||||
MP_ASSIGN_OR_RETURN(output_tensor, ConvertToTensor(fft_output_matrix,
|
ASSIGN_OR_RETURN(output_tensor, ConvertToTensor(fft_output_matrix,
|
||||||
{2, fft_size_ / 2}));
|
{2, fft_size_ / 2}));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case Options::WITH_DC_AND_NYQUIST: {
|
case Options::WITH_DC_AND_NYQUIST: {
|
||||||
|
@ -529,7 +528,7 @@ absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
|
||||||
// The last two elements are Nyquist component.
|
// The last two elements are Nyquist component.
|
||||||
fft_output_matrix(fft_size_) = fft_output_[1]; // Nyquist real part
|
fft_output_matrix(fft_size_) = fft_output_[1]; // Nyquist real part
|
||||||
fft_output_matrix(fft_size_ + 1) = 0.0f; // Nyquist imagery part
|
fft_output_matrix(fft_size_ + 1) = 0.0f; // Nyquist imagery part
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
output_tensor,
|
output_tensor,
|
||||||
ConvertToTensor(fft_output_matrix, {2, (fft_size_ + 2) / 2}));
|
ConvertToTensor(fft_output_matrix, {2, (fft_size_ + 2) / 2}));
|
||||||
break;
|
break;
|
||||||
|
@ -537,7 +536,7 @@ absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
|
||||||
case Options::WITHOUT_DC_AND_NYQUIST: {
|
case Options::WITHOUT_DC_AND_NYQUIST: {
|
||||||
Matrix fft_output_matrix =
|
Matrix fft_output_matrix =
|
||||||
Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
|
Eigen::Map<const Matrix>(fft_output_.data() + 2, 1, fft_size_ - 2);
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
output_tensor,
|
output_tensor,
|
||||||
ConvertToTensor(fft_output_matrix, {2, (fft_size_ - 2) / 2}));
|
ConvertToTensor(fft_output_matrix, {2, (fft_size_ - 2) / 2}));
|
||||||
break;
|
break;
|
||||||
|
@ -547,8 +546,8 @@ absl::Status AudioToTensorCalculator::OutputTensor(const Matrix& block,
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
MP_ASSIGN_OR_RETURN(output_tensor,
|
ASSIGN_OR_RETURN(output_tensor,
|
||||||
ConvertToTensor(block, {num_channels_, num_samples_}));
|
ConvertToTensor(block, {num_channels_, num_samples_}));
|
||||||
}
|
}
|
||||||
kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
|
kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
|
|
@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
|
||||||
// will be converted into tensors.
|
// will be converted into tensors.
|
||||||
optional double target_sample_rate = 4;
|
optional double target_sample_rate = 4;
|
||||||
|
|
||||||
// Whether to treat the input audio stream as a continuous stream or a batch
|
// Whether to treat the input audio stream as a continous stream or a batch
|
||||||
// of unrelated audio buffers.
|
// of unrelated audio buffers.
|
||||||
optional bool stream_mode = 5 [default = true];
|
optional bool stream_mode = 5 [default = true];
|
||||||
|
|
||||||
|
|
|
@ -68,6 +68,7 @@ constexpr absl::string_view kSeparatorToken = "[SEP]";
|
||||||
//
|
//
|
||||||
// This calculator is currently configured for the TextClassifier Task but it
|
// This calculator is currently configured for the TextClassifier Task but it
|
||||||
// will eventually be generalized for other Text Tasks.
|
// will eventually be generalized for other Text Tasks.
|
||||||
|
// TODO: Handle preprocessing for other Text Tasks too.
|
||||||
//
|
//
|
||||||
// Inputs:
|
// Inputs:
|
||||||
// TEXT - std::string
|
// TEXT - std::string
|
||||||
|
@ -160,9 +161,9 @@ absl::Status BertPreprocessorCalculator::Open(CalculatorContext* cc) {
|
||||||
&kMetadataExtractorSideIn(cc).Get();
|
&kMetadataExtractorSideIn(cc).Get();
|
||||||
const tflite::ProcessUnit* tokenizer_metadata =
|
const tflite::ProcessUnit* tokenizer_metadata =
|
||||||
metadata_extractor->GetInputProcessUnit(kTokenizerProcessUnitIndex);
|
metadata_extractor->GetInputProcessUnit(kTokenizerProcessUnitIndex);
|
||||||
MP_ASSIGN_OR_RETURN(tokenizer_,
|
ASSIGN_OR_RETURN(tokenizer_,
|
||||||
tasks::text::tokenizers::CreateTokenizerFromProcessUnit(
|
tasks::text::tokenizers::CreateTokenizerFromProcessUnit(
|
||||||
tokenizer_metadata, metadata_extractor));
|
tokenizer_metadata, metadata_extractor));
|
||||||
|
|
||||||
auto* input_tensors_metadata = metadata_extractor->GetInputTensorMetadata();
|
auto* input_tensors_metadata = metadata_extractor->GetInputTensorMetadata();
|
||||||
input_ids_tensor_index_ = FindTensorIndexByMetadataName(
|
input_ids_tensor_index_ = FindTensorIndexByMetadataName(
|
||||||
|
|
|
@ -67,10 +67,9 @@ absl::StatusOr<std::vector<std::vector<int>>> RunBertPreprocessorCalculator(
|
||||||
tool::AddVectorSink("tensors", &graph_config, &output_packets);
|
tool::AddVectorSink("tensors", &graph_config, &output_packets);
|
||||||
|
|
||||||
std::string model_buffer = tasks::core::LoadBinaryContent(model_path.data());
|
std::string model_buffer = tasks::core::LoadBinaryContent(model_path.data());
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
|
||||||
std::unique_ptr<ModelMetadataExtractor> metadata_extractor,
|
ModelMetadataExtractor::CreateFromModelBuffer(
|
||||||
ModelMetadataExtractor::CreateFromModelBuffer(model_buffer.data(),
|
model_buffer.data(), model_buffer.size()));
|
||||||
model_buffer.size()));
|
|
||||||
// Run the graph.
|
// Run the graph.
|
||||||
CalculatorGraph graph;
|
CalculatorGraph graph;
|
||||||
MP_RETURN_IF_ERROR(graph.Initialize(
|
MP_RETURN_IF_ERROR(graph.Initialize(
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
#include <utility>
|
#include <utility>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
|
#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator.pb.h"
|
#include "mediapipe/framework/calculator.pb.h"
|
||||||
#include "mediapipe/framework/calculator_framework.h"
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
@ -66,7 +65,7 @@ template <typename T>
|
||||||
Tensor MakeTensor(std::initializer_list<int> shape,
|
Tensor MakeTensor(std::initializer_list<int> shape,
|
||||||
std::initializer_list<T> values) {
|
std::initializer_list<T> values) {
|
||||||
Tensor tensor(TensorElementType<T>::value, shape);
|
Tensor tensor(TensorElementType<T>::value, shape);
|
||||||
ABSL_CHECK_EQ(values.size(), tensor.shape().num_elements())
|
CHECK_EQ(values.size(), tensor.shape().num_elements())
|
||||||
<< "The size of `values` is incompatible with `shape`";
|
<< "The size of `values` is incompatible with `shape`";
|
||||||
absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
|
absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
|
||||||
return tensor;
|
return tensor;
|
||||||
|
|
|
@ -16,7 +16,6 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
|
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
|
||||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||||
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
|
||||||
|
@ -82,7 +81,7 @@ namespace api2 {
|
||||||
//
|
//
|
||||||
// Outputs:
|
// Outputs:
|
||||||
// TENSORS - std::vector<Tensor>
|
// TENSORS - std::vector<Tensor>
|
||||||
// Vector containing a single Tensor populated with an extracted RGB image.
|
// Vector containing a single Tensor populated with an extrated RGB image.
|
||||||
// MATRIX - std::array<float, 16> @Optional
|
// MATRIX - std::array<float, 16> @Optional
|
||||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix that
|
// An std::array<float, 16> representing a 4x4 row-major-order matrix that
|
||||||
// maps a point on the input image to a point on the output tensor, and
|
// maps a point on the input image to a point on the output tensor, and
|
||||||
|
@ -192,19 +191,18 @@ class ImageToTensorCalculator : public Node {
|
||||||
}
|
}
|
||||||
|
|
||||||
#if MEDIAPIPE_DISABLE_GPU
|
#if MEDIAPIPE_DISABLE_GPU
|
||||||
MP_ASSIGN_OR_RETURN(auto image, GetInputImage(kIn(cc)));
|
ASSIGN_OR_RETURN(auto image, GetInputImage(kIn(cc)));
|
||||||
#else
|
#else
|
||||||
const bool is_input_gpu = kInGpu(cc).IsConnected();
|
const bool is_input_gpu = kInGpu(cc).IsConnected();
|
||||||
MP_ASSIGN_OR_RETURN(auto image, is_input_gpu ? GetInputImage(kInGpu(cc))
|
ASSIGN_OR_RETURN(auto image, is_input_gpu ? GetInputImage(kInGpu(cc))
|
||||||
: GetInputImage(kIn(cc)));
|
: GetInputImage(kIn(cc)));
|
||||||
#endif // MEDIAPIPE_DISABLE_GPU
|
#endif // MEDIAPIPE_DISABLE_GPU
|
||||||
|
|
||||||
RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect);
|
RotatedRect roi = GetRoi(image->width(), image->height(), norm_rect);
|
||||||
const int tensor_width = params_.output_width.value_or(image->width());
|
const int tensor_width = params_.output_width.value_or(image->width());
|
||||||
const int tensor_height = params_.output_height.value_or(image->height());
|
const int tensor_height = params_.output_height.value_or(image->height());
|
||||||
MP_ASSIGN_OR_RETURN(auto padding,
|
ASSIGN_OR_RETURN(auto padding, PadRoi(tensor_width, tensor_height,
|
||||||
PadRoi(tensor_width, tensor_height,
|
options_.keep_aspect_ratio(), &roi));
|
||||||
options_.keep_aspect_ratio(), &roi));
|
|
||||||
if (kOutLetterboxPadding(cc).IsConnected()) {
|
if (kOutLetterboxPadding(cc).IsConnected()) {
|
||||||
kOutLetterboxPadding(cc).Send(padding);
|
kOutLetterboxPadding(cc).Send(padding);
|
||||||
}
|
}
|
||||||
|
@ -212,7 +210,7 @@ class ImageToTensorCalculator : public Node {
|
||||||
std::array<float, 16> matrix;
|
std::array<float, 16> matrix;
|
||||||
GetRotatedSubRectToRectTransformMatrix(
|
GetRotatedSubRectToRectTransformMatrix(
|
||||||
roi, image->width(), image->height(),
|
roi, image->width(), image->height(),
|
||||||
/*flip_horizontally=*/false, &matrix);
|
/*flip_horizontaly=*/false, &matrix);
|
||||||
kOutMatrix(cc).Send(std::move(matrix));
|
kOutMatrix(cc).Send(std::move(matrix));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -248,20 +246,20 @@ class ImageToTensorCalculator : public Node {
|
||||||
if (!gpu_converter_) {
|
if (!gpu_converter_) {
|
||||||
#if !MEDIAPIPE_DISABLE_GPU
|
#if !MEDIAPIPE_DISABLE_GPU
|
||||||
#if MEDIAPIPE_METAL_ENABLED
|
#if MEDIAPIPE_METAL_ENABLED
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
gpu_converter_,
|
gpu_converter_,
|
||||||
CreateMetalConverter(cc, GetBorderMode(options_.border_mode())));
|
CreateMetalConverter(cc, GetBorderMode(options_.border_mode())));
|
||||||
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||||
MP_ASSIGN_OR_RETURN(gpu_converter_,
|
ASSIGN_OR_RETURN(gpu_converter_,
|
||||||
CreateImageToGlBufferTensorConverter(
|
CreateImageToGlBufferTensorConverter(
|
||||||
cc, DoesGpuInputStartAtBottom(options_),
|
cc, DoesGpuInputStartAtBottom(options_),
|
||||||
GetBorderMode(options_.border_mode())));
|
GetBorderMode(options_.border_mode())));
|
||||||
#else
|
#else
|
||||||
if (!gpu_converter_) {
|
if (!gpu_converter_) {
|
||||||
MP_ASSIGN_OR_RETURN(gpu_converter_,
|
ASSIGN_OR_RETURN(gpu_converter_,
|
||||||
CreateImageToGlTextureTensorConverter(
|
CreateImageToGlTextureTensorConverter(
|
||||||
cc, DoesGpuInputStartAtBottom(options_),
|
cc, DoesGpuInputStartAtBottom(options_),
|
||||||
GetBorderMode(options_.border_mode())));
|
GetBorderMode(options_.border_mode())));
|
||||||
}
|
}
|
||||||
if (!gpu_converter_) {
|
if (!gpu_converter_) {
|
||||||
return absl::UnimplementedError(
|
return absl::UnimplementedError(
|
||||||
|
@ -273,24 +271,22 @@ class ImageToTensorCalculator : public Node {
|
||||||
} else {
|
} else {
|
||||||
if (!cpu_converter_) {
|
if (!cpu_converter_) {
|
||||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(cpu_converter_,
|
||||||
cpu_converter_,
|
CreateOpenCvConverter(
|
||||||
CreateOpenCvConverter(
|
cc, GetBorderMode(options_.border_mode()),
|
||||||
cc, GetBorderMode(options_.border_mode()),
|
GetOutputTensorType(/*uses_gpu=*/false, params_)));
|
||||||
GetOutputTensorType(/*uses_gpu=*/false, params_)));
|
|
||||||
// TODO: FrameBuffer-based converter needs to call GetGpuBuffer()
|
// TODO: FrameBuffer-based converter needs to call GetGpuBuffer()
|
||||||
// to get access to a FrameBuffer view. Investigate if GetGpuBuffer() can be
|
// to get access to a FrameBuffer view. Investigate if GetGpuBuffer() can be
|
||||||
// made available even with MEDIAPIPE_DISABLE_GPU set.
|
// made available even with MEDIAPIPE_DISABLE_GPU set.
|
||||||
#elif MEDIAPIPE_ENABLE_HALIDE
|
#elif MEDIAPIPE_ENABLE_HALIDE
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(cpu_converter_,
|
||||||
cpu_converter_,
|
CreateFrameBufferConverter(
|
||||||
CreateFrameBufferConverter(
|
cc, GetBorderMode(options_.border_mode()),
|
||||||
cc, GetBorderMode(options_.border_mode()),
|
GetOutputTensorType(/*uses_gpu=*/false, params_)));
|
||||||
GetOutputTensorType(/*uses_gpu=*/false, params_)));
|
|
||||||
#else
|
#else
|
||||||
ABSL_LOG(FATAL) << "Cannot create image to tensor CPU converter since "
|
LOG(FATAL) << "Cannot create image to tensor CPU converter since "
|
||||||
"MEDIAPIPE_DISABLE_OPENCV is defined and "
|
"MEDIAPIPE_DISABLE_OPENCV is defined and "
|
||||||
"MEDIAPIPE_ENABLE_HALIDE is not defined.";
|
"MEDIAPIPE_ENABLE_HALIDE is not defined.";
|
||||||
#endif // !MEDIAPIPE_DISABLE_HALIDE
|
#endif // !MEDIAPIPE_DISABLE_HALIDE
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/flags/flag.h"
|
#include "absl/flags/flag.h"
|
||||||
#include "absl/log/absl_check.h"
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
#include "absl/strings/str_format.h"
|
#include "absl/strings/str_format.h"
|
||||||
#include "absl/strings/substitute.h"
|
#include "absl/strings/substitute.h"
|
||||||
|
@ -206,7 +205,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
|
||||||
} else if (image_channels == 1) {
|
} else if (image_channels == 1) {
|
||||||
return ImageFormat::GRAY8;
|
return ImageFormat::GRAY8;
|
||||||
}
|
}
|
||||||
ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
|
CHECK(false) << "Unsupported input image channles: " << image_channels;
|
||||||
}
|
}
|
||||||
|
|
||||||
Packet MakeImageFramePacket(cv::Mat input) {
|
Packet MakeImageFramePacket(cv::Mat input) {
|
||||||
|
|
|
@ -175,9 +175,9 @@ absl::Status FrameBufferProcessor::CropRotateResize90Degrees(
|
||||||
cropped_buffer_ = std::make_unique<uint8_t[]>(cropped_buffer_size);
|
cropped_buffer_ = std::make_unique<uint8_t[]>(cropped_buffer_size);
|
||||||
cropped_buffer_size_ = cropped_buffer_size;
|
cropped_buffer_size_ = cropped_buffer_size;
|
||||||
}
|
}
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(cropped,
|
||||||
cropped, frame_buffer::CreateFromRawBuffer(
|
frame_buffer::CreateFromRawBuffer(
|
||||||
cropped_buffer_.get(), cropped_dims, input->format()));
|
cropped_buffer_.get(), cropped_dims, input->format()));
|
||||||
}
|
}
|
||||||
MP_RETURN_IF_ERROR(
|
MP_RETURN_IF_ERROR(
|
||||||
frame_buffer::Crop(*input, left, top, right, bottom, cropped.get()));
|
frame_buffer::Crop(*input, left, top, right, bottom, cropped.get()));
|
||||||
|
@ -194,9 +194,9 @@ absl::Status FrameBufferProcessor::CropRotateResize90Degrees(
|
||||||
rotated_buffer_ = std::make_unique<uint8_t[]>(rotated_buffer_size);
|
rotated_buffer_ = std::make_unique<uint8_t[]>(rotated_buffer_size);
|
||||||
rotated_buffer_size_ = rotated_buffer_size;
|
rotated_buffer_size_ = rotated_buffer_size;
|
||||||
}
|
}
|
||||||
MP_ASSIGN_OR_RETURN(auto rotated, frame_buffer::CreateFromRawBuffer(
|
ASSIGN_OR_RETURN(auto rotated, frame_buffer::CreateFromRawBuffer(
|
||||||
rotated_buffer_.get(), rotated_dims,
|
rotated_buffer_.get(), rotated_dims,
|
||||||
cropped->format()));
|
cropped->format()));
|
||||||
}
|
}
|
||||||
MP_RETURN_IF_ERROR(
|
MP_RETURN_IF_ERROR(
|
||||||
frame_buffer::Rotate(*cropped, rotation_degrees, rotated.get()));
|
frame_buffer::Rotate(*cropped, rotation_degrees, rotated.get()));
|
||||||
|
@ -217,10 +217,9 @@ absl::Status FrameBufferProcessor::ConvertToFloatTensor(
|
||||||
RET_CHECK(output_tensor.element_type() == Tensor::ElementType::kFloat32);
|
RET_CHECK(output_tensor.element_type() == Tensor::ElementType::kFloat32);
|
||||||
constexpr float kInputImageRangeMin = 0.0f;
|
constexpr float kInputImageRangeMin = 0.0f;
|
||||||
constexpr float kInputImageRangeMax = 255.0f;
|
constexpr float kInputImageRangeMax = 255.0f;
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(auto transform, GetValueRangeTransformation(
|
||||||
auto transform,
|
kInputImageRangeMin, kInputImageRangeMax,
|
||||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
range_min, range_max));
|
||||||
range_min, range_max));
|
|
||||||
return frame_buffer::ToFloatTensor(*input_frame, transform.scale,
|
return frame_buffer::ToFloatTensor(*input_frame, transform.scale,
|
||||||
transform.offset, output_tensor);
|
transform.offset, output_tensor);
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,7 +57,7 @@ class SubRectExtractorGl {
|
||||||
absl::Status ExtractSubRectToBuffer(
|
absl::Status ExtractSubRectToBuffer(
|
||||||
const tflite::gpu::gl::GlTexture& texture,
|
const tflite::gpu::gl::GlTexture& texture,
|
||||||
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
|
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
|
||||||
bool flip_horizontally, float alpha, float beta,
|
bool flip_horizontaly, float alpha, float beta,
|
||||||
const tflite::gpu::HW& destination_size,
|
const tflite::gpu::HW& destination_size,
|
||||||
tflite::gpu::gl::CommandQueue* command_queue,
|
tflite::gpu::gl::CommandQueue* command_queue,
|
||||||
tflite::gpu::gl::GlBuffer* destination);
|
tflite::gpu::gl::GlBuffer* destination);
|
||||||
|
@ -154,13 +154,13 @@ void main() {
|
||||||
absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
|
absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
|
||||||
const tflite::gpu::gl::GlTexture& texture,
|
const tflite::gpu::gl::GlTexture& texture,
|
||||||
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
|
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
|
||||||
bool flip_horizontally, float alpha, float beta,
|
bool flip_horizontaly, float alpha, float beta,
|
||||||
const tflite::gpu::HW& destination_size,
|
const tflite::gpu::HW& destination_size,
|
||||||
tflite::gpu::gl::CommandQueue* command_queue,
|
tflite::gpu::gl::CommandQueue* command_queue,
|
||||||
tflite::gpu::gl::GlBuffer* destination) {
|
tflite::gpu::gl::GlBuffer* destination) {
|
||||||
std::array<float, 16> transform_mat;
|
std::array<float, 16> transform_mat;
|
||||||
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
|
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
|
||||||
texture_size.h, flip_horizontally,
|
texture_size.h, flip_horizontaly,
|
||||||
&transform_mat);
|
&transform_mat);
|
||||||
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
|
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
<< "OpenGL ES 3.1 is required.";
|
<< "OpenGL ES 3.1 is required.";
|
||||||
command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info);
|
command_queue_ = tflite::gpu::gl::NewCommandQueue(gpu_info);
|
||||||
|
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
auto extractor,
|
auto extractor,
|
||||||
SubRectExtractorGl::Create(gl_helper_.GetGlContext(),
|
SubRectExtractorGl::Create(gl_helper_.GetGlContext(),
|
||||||
input_starts_at_bottom, border_mode));
|
input_starts_at_bottom, border_mode));
|
||||||
|
@ -293,10 +293,10 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
|
|
||||||
constexpr float kInputImageRangeMin = 0.0f;
|
constexpr float kInputImageRangeMin = 0.0f;
|
||||||
constexpr float kInputImageRangeMax = 1.0f;
|
constexpr float kInputImageRangeMax = 1.0f;
|
||||||
MP_ASSIGN_OR_RETURN(auto transform,
|
ASSIGN_OR_RETURN(auto transform,
|
||||||
GetValueRangeTransformation(
|
GetValueRangeTransformation(kInputImageRangeMin,
|
||||||
kInputImageRangeMin, kInputImageRangeMax,
|
kInputImageRangeMax,
|
||||||
range_min, range_max));
|
range_min, range_max));
|
||||||
|
|
||||||
const int output_size = output_tensor.bytes() / output_shape.dims[0];
|
const int output_size = output_tensor.bytes() / output_shape.dims[0];
|
||||||
auto buffer_view = output_tensor.GetOpenGlBufferWriteView();
|
auto buffer_view = output_tensor.GetOpenGlBufferWriteView();
|
||||||
|
@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
input_texture,
|
input_texture,
|
||||||
tflite::gpu::HW(source_texture.height(), source_texture.width()),
|
tflite::gpu::HW(source_texture.height(), source_texture.width()),
|
||||||
roi,
|
roi,
|
||||||
/*flip_horizontally=*/false, transform.scale, transform.offset,
|
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||||
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
||||||
command_queue_.get(), &output));
|
command_queue_.get(), &output));
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,6 @@
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "absl/strings/str_cat.h"
|
#include "absl/strings/str_cat.h"
|
||||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
|
||||||
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
|
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
|
||||||
|
@ -193,13 +192,13 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
|
|
||||||
constexpr float kInputImageRangeMin = 0.0f;
|
constexpr float kInputImageRangeMin = 0.0f;
|
||||||
constexpr float kInputImageRangeMax = 1.0f;
|
constexpr float kInputImageRangeMax = 1.0f;
|
||||||
MP_ASSIGN_OR_RETURN(auto transform,
|
ASSIGN_OR_RETURN(auto transform,
|
||||||
GetValueRangeTransformation(
|
GetValueRangeTransformation(kInputImageRangeMin,
|
||||||
kInputImageRangeMin, kInputImageRangeMax,
|
kInputImageRangeMax,
|
||||||
range_min, range_max));
|
range_min, range_max));
|
||||||
auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
|
auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
|
||||||
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
|
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
|
||||||
/*flip_horizontally=*/false,
|
/*flip_horizontaly=*/false,
|
||||||
transform.scale, transform.offset,
|
transform.scale, transform.offset,
|
||||||
output_shape, &tensor_view));
|
output_shape, &tensor_view));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -210,7 +209,7 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
|
|
||||||
absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
|
absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
|
||||||
const RotatedRect& sub_rect,
|
const RotatedRect& sub_rect,
|
||||||
bool flip_horizontally, float alpha, float beta,
|
bool flip_horizontaly, float alpha, float beta,
|
||||||
const Tensor::Shape& output_shape,
|
const Tensor::Shape& output_shape,
|
||||||
Tensor::OpenGlTexture2dView* output) {
|
Tensor::OpenGlTexture2dView* output) {
|
||||||
const int output_height = output_shape.dims[1];
|
const int output_height = output_shape.dims[1];
|
||||||
|
@ -260,16 +259,16 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
// error. So in that case, we'll grab the transpose of our original matrix
|
// error. So in that case, we'll grab the transpose of our original matrix
|
||||||
// and send that instead.
|
// and send that instead.
|
||||||
const auto gl_context = mediapipe::GlContext::GetCurrent();
|
const auto gl_context = mediapipe::GlContext::GetCurrent();
|
||||||
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
|
LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
|
||||||
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
|
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
|
||||||
GetTransposedRotatedSubRectToRectTransformMatrix(
|
GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||||
sub_rect, texture.width(), texture.height(), flip_horizontally,
|
sub_rect, texture.width(), texture.height(), flip_horizontaly,
|
||||||
&transform_mat);
|
&transform_mat);
|
||||||
glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
|
glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
|
||||||
} else {
|
} else {
|
||||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
|
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
|
||||||
texture.height(),
|
texture.height(), flip_horizontaly,
|
||||||
flip_horizontally, &transform_mat);
|
&transform_mat);
|
||||||
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
|
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -304,7 +303,6 @@ class GlProcessor : public ImageToTensorConverter {
|
||||||
glBindTexture(GL_TEXTURE_2D, 0);
|
glBindTexture(GL_TEXTURE_2D, 0);
|
||||||
glActiveTexture(GL_TEXTURE0);
|
glActiveTexture(GL_TEXTURE0);
|
||||||
glBindTexture(GL_TEXTURE_2D, 0);
|
glBindTexture(GL_TEXTURE_2D, 0);
|
||||||
glFlush();
|
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
|
@ -179,13 +179,13 @@ class SubRectExtractorMetal {
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status Execute(id<MTLTexture> input_texture,
|
absl::Status Execute(id<MTLTexture> input_texture,
|
||||||
const RotatedRect& sub_rect, bool flip_horizontally,
|
const RotatedRect& sub_rect, bool flip_horizontaly,
|
||||||
float alpha, float beta,
|
float alpha, float beta,
|
||||||
const tflite::gpu::HW& destination_size,
|
const tflite::gpu::HW& destination_size,
|
||||||
id<MTLCommandBuffer> command_buffer,
|
id<MTLCommandBuffer> command_buffer,
|
||||||
id<MTLBuffer> destination) {
|
id<MTLBuffer> destination) {
|
||||||
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
|
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
|
||||||
return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
|
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
|
||||||
beta, destination_size, command_buffer,
|
beta, destination_size, command_buffer,
|
||||||
output_texture);
|
output_texture);
|
||||||
}
|
}
|
||||||
|
@ -211,7 +211,7 @@ class SubRectExtractorMetal {
|
||||||
|
|
||||||
absl::Status InternalExecute(id<MTLTexture> input_texture,
|
absl::Status InternalExecute(id<MTLTexture> input_texture,
|
||||||
const RotatedRect& sub_rect,
|
const RotatedRect& sub_rect,
|
||||||
bool flip_horizontally, float alpha, float beta,
|
bool flip_horizontaly, float alpha, float beta,
|
||||||
const tflite::gpu::HW& destination_size,
|
const tflite::gpu::HW& destination_size,
|
||||||
id<MTLCommandBuffer> command_buffer,
|
id<MTLCommandBuffer> command_buffer,
|
||||||
id<MTLTexture> output_texture) {
|
id<MTLTexture> output_texture) {
|
||||||
|
@ -223,7 +223,7 @@ class SubRectExtractorMetal {
|
||||||
std::array<float, 16> transform_mat;
|
std::array<float, 16> transform_mat;
|
||||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
|
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
|
||||||
input_texture.height,
|
input_texture.height,
|
||||||
flip_horizontally, &transform_mat);
|
flip_horizontaly, &transform_mat);
|
||||||
id<MTLBuffer> transform_mat_buffer =
|
id<MTLBuffer> transform_mat_buffer =
|
||||||
[device_ newBufferWithBytes:&transform_mat
|
[device_ newBufferWithBytes:&transform_mat
|
||||||
length:sizeof(transform_mat)
|
length:sizeof(transform_mat)
|
||||||
|
@ -345,9 +345,9 @@ class MetalProcessor : public ImageToTensorConverter {
|
||||||
absl::Status Init(CalculatorContext* cc, BorderMode border_mode) {
|
absl::Status Init(CalculatorContext* cc, BorderMode border_mode) {
|
||||||
metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc];
|
||||||
RET_CHECK(metal_helper_);
|
RET_CHECK(metal_helper_);
|
||||||
MP_ASSIGN_OR_RETURN(extractor_, SubRectExtractorMetal::Make(
|
ASSIGN_OR_RETURN(extractor_, SubRectExtractorMetal::Make(
|
||||||
metal_helper_.mtlDevice,
|
metal_helper_.mtlDevice,
|
||||||
OutputFormat::kF32C4, border_mode));
|
OutputFormat::kF32C4, border_mode));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -373,7 +373,7 @@ class MetalProcessor : public ImageToTensorConverter {
|
||||||
|
|
||||||
constexpr float kInputImageRangeMin = 0.0f;
|
constexpr float kInputImageRangeMin = 0.0f;
|
||||||
constexpr float kInputImageRangeMax = 1.0f;
|
constexpr float kInputImageRangeMax = 1.0f;
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
auto transform,
|
auto transform,
|
||||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
||||||
range_min, range_max));
|
range_min, range_max));
|
||||||
|
@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
|
||||||
MtlBufferView::GetWriteView(output_tensor, command_buffer);
|
MtlBufferView::GetWriteView(output_tensor, command_buffer);
|
||||||
MP_RETURN_IF_ERROR(extractor_->Execute(
|
MP_RETURN_IF_ERROR(extractor_->Execute(
|
||||||
texture, roi,
|
texture, roi,
|
||||||
/*flip_horizontally=*/false, transform.scale, transform.offset,
|
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||||
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
||||||
command_buffer, buffer_view.buffer()));
|
command_buffer, buffer_view.buffer()));
|
||||||
[command_buffer commit];
|
[command_buffer commit];
|
||||||
|
|
|
@ -159,7 +159,7 @@ class OpenCvProcessor : public ImageToTensorConverter {
|
||||||
|
|
||||||
constexpr float kInputImageRangeMin = 0.0f;
|
constexpr float kInputImageRangeMin = 0.0f;
|
||||||
constexpr float kInputImageRangeMax = 255.0f;
|
constexpr float kInputImageRangeMax = 255.0f;
|
||||||
MP_ASSIGN_OR_RETURN(
|
ASSIGN_OR_RETURN(
|
||||||
auto transform,
|
auto transform,
|
||||||
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax,
|
||||||
range_min, range_max));
|
range_min, range_max));
|
||||||
|
|
|
@ -92,7 +92,7 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
||||||
|
|
||||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||||
int rect_width, int rect_height,
|
int rect_width, int rect_height,
|
||||||
bool flip_horizontally,
|
bool flip_horizontaly,
|
||||||
std::array<float, 16>* matrix_ptr) {
|
std::array<float, 16>* matrix_ptr) {
|
||||||
std::array<float, 16>& matrix = *matrix_ptr;
|
std::array<float, 16>& matrix = *matrix_ptr;
|
||||||
// The resulting matrix is multiplication of below commented out matrices:
|
// The resulting matrix is multiplication of below commented out matrices:
|
||||||
|
@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||||
// {0.0f, 0.0f, a, 0.0f}
|
// {0.0f, 0.0f, a, 0.0f}
|
||||||
// {0.0f, 0.0f, 0.0f, 1.0f}
|
// {0.0f, 0.0f, 0.0f, 1.0f}
|
||||||
|
|
||||||
const float flip = flip_horizontally ? -1 : 1;
|
const float flip = flip_horizontaly ? -1 : 1;
|
||||||
// Matrix for optional horizontal flip around middle of output image.
|
// Matrix for optional horizontal flip around middle of output image.
|
||||||
// { fl , 0.0f, 0.0f, 0.0f}
|
// { fl , 0.0f, 0.0f, 0.0f}
|
||||||
// { 0.0f, 1.0f, 0.0f, 0.0f}
|
// { 0.0f, 1.0f, 0.0f, 0.0f}
|
||||||
|
@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||||
|
|
||||||
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||||
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
||||||
bool flip_horizontally, std::array<float, 16>* matrix_ptr) {
|
bool flip_horizontaly, std::array<float, 16>* matrix_ptr) {
|
||||||
std::array<float, 16>& matrix = *matrix_ptr;
|
std::array<float, 16>& matrix = *matrix_ptr;
|
||||||
// See comments in GetRotatedSubRectToRectTransformMatrix for detailed
|
// See comments in GetRotatedSubRectToRectTransformMatrix for detailed
|
||||||
// calculations.
|
// calculations.
|
||||||
const float a = sub_rect.width;
|
const float a = sub_rect.width;
|
||||||
const float b = sub_rect.height;
|
const float b = sub_rect.height;
|
||||||
const float flip = flip_horizontally ? -1 : 1;
|
const float flip = flip_horizontaly ? -1 : 1;
|
||||||
const float c = std::cos(sub_rect.rotation);
|
const float c = std::cos(sub_rect.rotation);
|
||||||
const float d = std::sin(sub_rect.rotation);
|
const float d = std::sin(sub_rect.rotation);
|
||||||
const float e = sub_rect.center_x;
|
const float e = sub_rect.center_x;
|
||||||
|
|
|
@ -74,7 +74,7 @@ absl::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
|
||||||
// Represents a transformation of value which involves scaling and offsetting.
|
// Represents a transformation of value which involves scaling and offsetting.
|
||||||
// To apply transformation:
|
// To apply transformation:
|
||||||
// ValueTransformation transform = ...
|
// ValueTransformation transform = ...
|
||||||
// float transformed_value = transform.scale * value + transform.offset;
|
// float transformed_value = transform.scale * value + transfrom.offset;
|
||||||
struct ValueTransformation {
|
struct ValueTransformation {
|
||||||
float scale;
|
float scale;
|
||||||
float offset;
|
float offset;
|
||||||
|
@ -99,11 +99,11 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
||||||
// @sub_rect - rotated sub rect in absolute coordinates
|
// @sub_rect - rotated sub rect in absolute coordinates
|
||||||
// @rect_width - rect width
|
// @rect_width - rect width
|
||||||
// @rect_height - rect height
|
// @rect_height - rect height
|
||||||
// @flip_horizontally - we need to flip the output buffer.
|
// @flip_horizontaly - we need to flip the output buffer.
|
||||||
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
||||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||||
int rect_width, int rect_height,
|
int rect_width, int rect_height,
|
||||||
bool flip_horizontally,
|
bool flip_horizontaly,
|
||||||
std::array<float, 16>* matrix);
|
std::array<float, 16>* matrix);
|
||||||
|
|
||||||
// Returns the transpose of the matrix found with
|
// Returns the transpose of the matrix found with
|
||||||
|
@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||||
// @sub_rect - rotated sub rect in absolute coordinates
|
// @sub_rect - rotated sub rect in absolute coordinates
|
||||||
// @rect_width - rect width
|
// @rect_width - rect width
|
||||||
// @rect_height - rect height
|
// @rect_height - rect height
|
||||||
// @flip_horizontally - we need to flip the output buffer.
|
// @flip_horizontaly - we need to flip the output buffer.
|
||||||
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
||||||
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||||
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
||||||
bool flip_horizontally, std::array<float, 16>* matrix);
|
bool flip_horizontaly, std::array<float, 16>* matrix);
|
||||||
|
|
||||||
// Validates the output dimensions set in the option proto. The input option
|
// Validates the output dimensions set in the option proto. The input option
|
||||||
// proto is expected to have to following fields:
|
// proto is expected to have to following fields:
|
||||||
|
|
|
@ -88,20 +88,6 @@ message InferenceCalculatorOptions {
|
||||||
// serialized model is invalid or missing.
|
// serialized model is invalid or missing.
|
||||||
optional string serialized_model_dir = 7;
|
optional string serialized_model_dir = 7;
|
||||||
|
|
||||||
enum CacheWritingBehavior {
|
|
||||||
// Do not write any caches.
|
|
||||||
NO_WRITE = 0;
|
|
||||||
|
|
||||||
// Try to write caches, log on failure.
|
|
||||||
TRY_WRITE = 1;
|
|
||||||
|
|
||||||
// Write caches or return an error if write fails.
|
|
||||||
WRITE_OR_ERROR = 2;
|
|
||||||
}
|
|
||||||
// Specifies how GPU caches are written to disk.
|
|
||||||
optional CacheWritingBehavior cache_writing_behavior = 10
|
|
||||||
[default = WRITE_OR_ERROR];
|
|
||||||
|
|
||||||
// Unique token identifying the model. Used in conjunction with
|
// Unique token identifying the model. Used in conjunction with
|
||||||
// "serialized_model_dir". It is the caller's responsibility to ensure
|
// "serialized_model_dir". It is the caller's responsibility to ensure
|
||||||
// there is no clash of the tokens.
|
// there is no clash of the tokens.
|
||||||
|
|
|
@ -60,7 +60,7 @@ absl::Status InferenceCalculatorCpuImpl::UpdateContract(
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status InferenceCalculatorCpuImpl::Open(CalculatorContext* cc) {
|
absl::Status InferenceCalculatorCpuImpl::Open(CalculatorContext* cc) {
|
||||||
MP_ASSIGN_OR_RETURN(inference_runner_, CreateInferenceRunner(cc));
|
ASSIGN_OR_RETURN(inference_runner_, CreateInferenceRunner(cc));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,8 +71,8 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
|
||||||
const auto& input_tensors = *kInTensors(cc);
|
const auto& input_tensors = *kInTensors(cc);
|
||||||
RET_CHECK(!input_tensors.empty());
|
RET_CHECK(!input_tensors.empty());
|
||||||
|
|
||||||
MP_ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
|
ASSIGN_OR_RETURN(std::vector<Tensor> output_tensors,
|
||||||
inference_runner_->Run(cc, input_tensors));
|
inference_runner_->Run(cc, input_tensors));
|
||||||
kOutTensors(cc).Send(std::move(output_tensors));
|
kOutTensors(cc).Send(std::move(output_tensors));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
@ -84,11 +84,11 @@ absl::Status InferenceCalculatorCpuImpl::Close(CalculatorContext* cc) {
|
||||||
|
|
||||||
absl::StatusOr<std::unique_ptr<InferenceRunner>>
|
absl::StatusOr<std::unique_ptr<InferenceRunner>>
|
||||||
InferenceCalculatorCpuImpl::CreateInferenceRunner(CalculatorContext* cc) {
|
InferenceCalculatorCpuImpl::CreateInferenceRunner(CalculatorContext* cc) {
|
||||||
MP_ASSIGN_OR_RETURN(auto model_packet, GetModelAsPacket(cc));
|
ASSIGN_OR_RETURN(auto model_packet, GetModelAsPacket(cc));
|
||||||
MP_ASSIGN_OR_RETURN(auto op_resolver_packet, GetOpResolverAsPacket(cc));
|
ASSIGN_OR_RETURN(auto op_resolver_packet, GetOpResolverAsPacket(cc));
|
||||||
const int interpreter_num_threads =
|
const int interpreter_num_threads =
|
||||||
cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread();
|
cc->Options<mediapipe::InferenceCalculatorOptions>().cpu_num_thread();
|
||||||
MP_ASSIGN_OR_RETURN(TfLiteDelegatePtr delegate, MaybeCreateDelegate(cc));
|
ASSIGN_OR_RETURN(TfLiteDelegatePtr delegate, MaybeCreateDelegate(cc));
|
||||||
return CreateInferenceInterpreterDelegateRunner(
|
return CreateInferenceInterpreterDelegateRunner(
|
||||||
std::move(model_packet), std::move(op_resolver_packet),
|
std::move(model_packet), std::move(op_resolver_packet),
|
||||||
std::move(delegate), interpreter_num_threads);
|
std::move(delegate), interpreter_num_threads);
|
||||||
|
|
|
@ -100,7 +100,7 @@ absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Init(
|
||||||
|
|
||||||
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadModel(
|
absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadModel(
|
||||||
CalculatorContext* cc) {
|
CalculatorContext* cc) {
|
||||||
MP_ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
||||||
const auto& model = *model_packet_.Get();
|
const auto& model = *model_packet_.Get();
|
||||||
if (kSideInOpResolver(cc).IsConnected()) {
|
if (kSideInOpResolver(cc).IsConnected()) {
|
||||||
const tflite::OpResolver& op_resolver = kSideInOpResolver(cc).Get();
|
const tflite::OpResolver& op_resolver = kSideInOpResolver(cc).Get();
|
||||||
|
|
|
@ -12,7 +12,6 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
@ -27,7 +26,6 @@
|
||||||
#include "mediapipe/util/tflite/tflite_gpu_runner.h"
|
#include "mediapipe/util/tflite/tflite_gpu_runner.h"
|
||||||
|
|
||||||
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
|
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "mediapipe/framework/deps/file_path.h"
|
#include "mediapipe/framework/deps/file_path.h"
|
||||||
#include "mediapipe/util/android/file/base/file.h"
|
#include "mediapipe/util/android/file/base/file.h"
|
||||||
#include "mediapipe/util/android/file/base/filesystem.h"
|
#include "mediapipe/util/android/file/base/filesystem.h"
|
||||||
|
@ -70,21 +68,13 @@ class InferenceCalculatorGlAdvancedImpl
|
||||||
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
|
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
|
||||||
gpu_delegate_options);
|
gpu_delegate_options);
|
||||||
absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
||||||
// Writes caches to disk based on |cache_writing_behavior_|.
|
|
||||||
absl::Status SaveGpuCachesBasedOnBehavior(
|
|
||||||
tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
|
||||||
bool UseSerializedModel() const { return use_serialized_model_; }
|
|
||||||
|
|
||||||
private:
|
|
||||||
// Writes caches to disk, returns error on failure.
|
|
||||||
absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
|
||||||
|
|
||||||
|
private:
|
||||||
bool use_kernel_caching_ = false;
|
bool use_kernel_caching_ = false;
|
||||||
std::string cached_kernel_filename_;
|
std::string cached_kernel_filename_;
|
||||||
bool use_serialized_model_ = false;
|
bool use_serialized_model_ = false;
|
||||||
std::string serialized_model_path_;
|
std::string serialized_model_path_;
|
||||||
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::CacheWritingBehavior
|
|
||||||
cache_writing_behavior_;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Helper class that wraps everything related to GPU inference acceleration.
|
// Helper class that wraps everything related to GPU inference acceleration.
|
||||||
|
@ -160,6 +150,8 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
|
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
|
||||||
|
MP_RETURN_IF_ERROR(
|
||||||
|
on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
|
||||||
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
|
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
|
||||||
tflite_gpu_runner_.reset();
|
tflite_gpu_runner_.reset();
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -170,7 +162,7 @@ absl::Status
|
||||||
InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
|
InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
|
||||||
CalculatorContext* cc,
|
CalculatorContext* cc,
|
||||||
const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
|
const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
|
||||||
MP_ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
||||||
const auto& model = *model_packet_.Get();
|
const auto& model = *model_packet_.Get();
|
||||||
|
|
||||||
bool allow_precision_loss = delegate.gpu().allow_precision_loss();
|
bool allow_precision_loss = delegate.gpu().allow_precision_loss();
|
||||||
|
@ -234,15 +226,9 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
|
||||||
tflite_gpu_runner_->GetOutputShapes()[i].c};
|
tflite_gpu_runner_->GetOutputShapes()[i].c};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (on_disk_cache_helper_.UseSerializedModel()) {
|
|
||||||
tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel();
|
|
||||||
}
|
|
||||||
|
|
||||||
MP_RETURN_IF_ERROR(
|
MP_RETURN_IF_ERROR(
|
||||||
on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
|
on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
|
||||||
MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build());
|
return tflite_gpu_runner_->Build();
|
||||||
return on_disk_cache_helper_.SaveGpuCachesBasedOnBehavior(
|
|
||||||
tflite_gpu_runner_.get());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
|
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
|
||||||
|
@ -271,51 +257,24 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
|
||||||
mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
|
mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
|
||||||
gpu_delegate_options.model_token());
|
gpu_delegate_options.model_token());
|
||||||
}
|
}
|
||||||
cache_writing_behavior_ = gpu_delegate_options.has_cache_writing_behavior()
|
|
||||||
? gpu_delegate_options.cache_writing_behavior()
|
|
||||||
: mediapipe::InferenceCalculatorOptions::
|
|
||||||
Delegate::Gpu::WRITE_OR_ERROR;
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::
|
|
||||||
SaveGpuCachesBasedOnBehavior(
|
|
||||||
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
|
||||||
switch (cache_writing_behavior_) {
|
|
||||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::NO_WRITE:
|
|
||||||
return absl::OkStatus();
|
|
||||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::TRY_WRITE: {
|
|
||||||
auto status = SaveGpuCaches(gpu_runner);
|
|
||||||
if (!status.ok()) {
|
|
||||||
ABSL_LOG_FIRST_N(WARNING, 1) << "Failed to save gpu caches: " << status;
|
|
||||||
}
|
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::WRITE_OR_ERROR:
|
|
||||||
return SaveGpuCaches(gpu_runner);
|
|
||||||
default:
|
|
||||||
ABSL_LOG_FIRST_N(ERROR, 1)
|
|
||||||
<< "Unknown cache writing behavior: "
|
|
||||||
<< static_cast<uint32_t>(cache_writing_behavior_);
|
|
||||||
return absl::InvalidArgumentError("Unknown cache writing behavior.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status
|
absl::Status
|
||||||
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
|
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
|
||||||
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
||||||
if (use_kernel_caching_) {
|
if (use_kernel_caching_) {
|
||||||
// Save kernel file.
|
// Save kernel file.
|
||||||
MP_ASSIGN_OR_RETURN(std::vector<uint8_t> kernel_cache,
|
ASSIGN_OR_RETURN(std::vector<uint8_t> kernel_cache,
|
||||||
gpu_runner->GetSerializedBinaryCache());
|
gpu_runner->GetSerializedBinaryCache());
|
||||||
std::string cache_str(kernel_cache.begin(), kernel_cache.end());
|
std::string cache_str(kernel_cache.begin(), kernel_cache.end());
|
||||||
MP_RETURN_IF_ERROR(
|
MP_RETURN_IF_ERROR(
|
||||||
mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
|
mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
|
||||||
}
|
}
|
||||||
if (use_serialized_model_) {
|
if (use_serialized_model_) {
|
||||||
// Save serialized model file.
|
// Save serialized model file.
|
||||||
MP_ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
|
ASSIGN_OR_RETURN(std::vector<uint8_t> serialized_model_vec,
|
||||||
gpu_runner->GetSerializedModel());
|
gpu_runner->GetSerializedModel());
|
||||||
absl::string_view serialized_model(
|
absl::string_view serialized_model(
|
||||||
reinterpret_cast<char*>(serialized_model_vec.data()),
|
reinterpret_cast<char*>(serialized_model_vec.data()),
|
||||||
serialized_model_vec.size());
|
serialized_model_vec.size());
|
||||||
|
@ -355,12 +314,6 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::
|
|
||||||
SaveGpuCachesBasedOnBehavior(
|
|
||||||
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
|
||||||
return absl::OkStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
absl::Status
|
absl::Status
|
||||||
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
|
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
|
||||||
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
|
||||||
|
@ -412,8 +365,8 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
|
||||||
RET_CHECK(!input_tensors.empty());
|
RET_CHECK(!input_tensors.empty());
|
||||||
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
|
auto output_tensors = absl::make_unique<std::vector<Tensor>>();
|
||||||
|
|
||||||
MP_ASSIGN_OR_RETURN(*output_tensors,
|
ASSIGN_OR_RETURN(*output_tensors,
|
||||||
gpu_inference_runner_->Process(cc, input_tensors));
|
gpu_inference_runner_->Process(cc, input_tensors));
|
||||||
|
|
||||||
kOutTensors(cc).Send(std::move(output_tensors));
|
kOutTensors(cc).Send(std::move(output_tensors));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
|
|
@ -21,7 +21,6 @@
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "absl/log/absl_log.h"
|
|
||||||
#include "absl/memory/memory.h"
|
#include "absl/memory/memory.h"
|
||||||
#include "absl/strings/str_format.h"
|
#include "absl/strings/str_format.h"
|
||||||
#include "mediapipe/calculators/tensor/inference_calculator.h"
|
#include "mediapipe/calculators/tensor/inference_calculator.h"
|
||||||
|
@ -75,7 +74,7 @@ tflite::gpu::BHWC BhwcFromTensorShape(const Tensor::Shape& shape) {
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
// Handles 0 and >4.
|
// Handles 0 and >4.
|
||||||
ABSL_LOG(FATAL)
|
LOG(FATAL)
|
||||||
<< "Dimensions size must be in range [1,4] for GPU inference, but "
|
<< "Dimensions size must be in range [1,4] for GPU inference, but "
|
||||||
<< shape.dims.size() << " is provided";
|
<< shape.dims.size() << " is provided";
|
||||||
}
|
}
|
||||||
|
@ -191,11 +190,6 @@ absl::Status InferenceCalculatorMetalImpl::Process(CalculatorContext* cc) {
|
||||||
[output_encoder endEncoding];
|
[output_encoder endEncoding];
|
||||||
}
|
}
|
||||||
[command_buffer commit];
|
[command_buffer commit];
|
||||||
// The below call is found (manual testing) to resolve flickering issues for
|
|
||||||
// some use cases where multiple Metal calculators are involved.
|
|
||||||
// TODO: investigate and ensure proper synchronization
|
|
||||||
// (e.g. fences/barriers/events).
|
|
||||||
[command_buffer waitUntilScheduled];
|
|
||||||
|
|
||||||
kOutTensors(cc).Send(std::move(output_tensors));
|
kOutTensors(cc).Send(std::move(output_tensors));
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
|
@ -213,9 +207,9 @@ absl::Status InferenceCalculatorMetalImpl::Close(CalculatorContext* cc) {
|
||||||
|
|
||||||
absl::Status InferenceCalculatorMetalImpl::InitInterpreter(
|
absl::Status InferenceCalculatorMetalImpl::InitInterpreter(
|
||||||
CalculatorContext* cc) {
|
CalculatorContext* cc) {
|
||||||
MP_ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
|
||||||
const auto& model = *model_packet_.Get();
|
const auto& model = *model_packet_.Get();
|
||||||
MP_ASSIGN_OR_RETURN(auto op_resolver_packet, GetOpResolverAsPacket(cc));
|
ASSIGN_OR_RETURN(auto op_resolver_packet, GetOpResolverAsPacket(cc));
|
||||||
const auto& op_resolver = op_resolver_packet.Get();
|
const auto& op_resolver = op_resolver_packet.Get();
|
||||||
tflite::InterpreterBuilder interpreter_builder(model, op_resolver);
|
tflite::InterpreterBuilder interpreter_builder(model, op_resolver);
|
||||||
AddDelegate(cc, &interpreter_builder);
|
AddDelegate(cc, &interpreter_builder);
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user