Fixed function names in MPPImage Utils
This commit is contained in:
commit
04f826e9d3
3
.bazelrc
3
.bazelrc
|
@ -98,6 +98,9 @@ build:darwin_arm64 --apple_platform_type=macos
|
|||
build:darwin_arm64 --macos_minimum_os=10.16
|
||||
build:darwin_arm64 --cpu=darwin_arm64
|
||||
|
||||
# Turn off maximum stdout size
|
||||
build --experimental_ui_max_stdouterr_bytes=-1
|
||||
|
||||
# This bazelrc file is meant to be written by a setup script.
|
||||
try-import %workspace%/.configure.bazelrc
|
||||
|
||||
|
|
|
@ -513,6 +513,9 @@ http_archive(
|
|||
"@//third_party:org_tensorflow_system_python.diff",
|
||||
# Diff is generated with a script, don't update it manually.
|
||||
"@//third_party:org_tensorflow_custom_ops.diff",
|
||||
# Works around Bazel issue with objc_library.
|
||||
# See https://github.com/bazelbuild/bazel/issues/19912
|
||||
"@//third_party:org_tensorflow_objc_build_fixes.diff",
|
||||
],
|
||||
patch_args = [
|
||||
"-p1",
|
||||
|
|
|
@ -0,0 +1,342 @@
|
|||
// !$*UTF8*$!
|
||||
{
|
||||
archiveVersion = 1;
|
||||
classes = {
|
||||
};
|
||||
objectVersion = 56;
|
||||
objects = {
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFrameworksBuildPhase section */
|
||||
8566B5562ABABF9A00AAB22A /* Frameworks */ = {
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXFrameworksBuildPhase section */
|
||||
|
||||
/* Begin PBXGroup section */
|
||||
8566B54F2ABABF9A00AAB22A = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
|
||||
8566B55A2ABABF9A00AAB22A /* Products */,
|
||||
);
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8566B55A2ABABF9A00AAB22A /* Products */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */,
|
||||
);
|
||||
path = MediaPipeTasksDocGen;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
/* End PBXGroup section */
|
||||
|
||||
/* Begin PBXHeadersBuildPhase section */
|
||||
8566B5542ABABF9A00AAB22A /* Headers */ = {
|
||||
isa = PBXHeadersBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXHeadersBuildPhase section */
|
||||
|
||||
/* Begin PBXNativeTarget section */
|
||||
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */;
|
||||
buildPhases = (
|
||||
8566B5542ABABF9A00AAB22A /* Headers */,
|
||||
8566B5552ABABF9A00AAB22A /* Sources */,
|
||||
8566B5562ABABF9A00AAB22A /* Frameworks */,
|
||||
8566B5572ABABF9A00AAB22A /* Resources */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
);
|
||||
name = MediaPipeTasksDocGen;
|
||||
productName = MediaPipeTasksDocGen;
|
||||
productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */;
|
||||
productType = "com.apple.product-type.framework";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
|
||||
/* Begin PBXProject section */
|
||||
8566B5502ABABF9A00AAB22A /* Project object */ = {
|
||||
isa = PBXProject;
|
||||
attributes = {
|
||||
BuildIndependentTargetsInParallel = 1;
|
||||
LastUpgradeCheck = 1430;
|
||||
TargetAttributes = {
|
||||
8566B5582ABABF9A00AAB22A = {
|
||||
CreatedOnToolsVersion = 14.3.1;
|
||||
};
|
||||
};
|
||||
};
|
||||
buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */;
|
||||
compatibilityVersion = "Xcode 14.0";
|
||||
developmentRegion = en;
|
||||
hasScannedForEncodings = 0;
|
||||
knownRegions = (
|
||||
en,
|
||||
Base,
|
||||
);
|
||||
mainGroup = 8566B54F2ABABF9A00AAB22A;
|
||||
productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */;
|
||||
projectDirPath = "";
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
|
||||
/* Begin PBXResourcesBuildPhase section */
|
||||
8566B5572ABABF9A00AAB22A /* Resources */ = {
|
||||
isa = PBXResourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXResourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXSourcesBuildPhase section */
|
||||
8566B5552ABABF9A00AAB22A /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXSourcesBuildPhase section */
|
||||
|
||||
/* Begin XCBuildConfiguration section */
|
||||
8566B55E2ABABF9A00AAB22A /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEBUG_INFORMATION_FORMAT = dwarf;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
ENABLE_TESTABILITY = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_DYNAMIC_NO_PIC = NO;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_OPTIMIZATION_LEVEL = 0;
|
||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
||||
"DEBUG=1",
|
||||
"$(inherited)",
|
||||
);
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
||||
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
||||
MTL_FAST_MATH = YES;
|
||||
ONLY_ACTIVE_ARCH = YES;
|
||||
SDKROOT = iphoneos;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
||||
VERSIONING_SYSTEM = "apple-generic";
|
||||
VERSION_INFO_PREFIX = "";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8566B55F2ABABF9A00AAB22A /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
||||
ENABLE_NS_ASSERTIONS = NO;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
|
||||
MTL_ENABLE_DEBUG_INFO = NO;
|
||||
MTL_FAST_MATH = YES;
|
||||
SDKROOT = iphoneos;
|
||||
SWIFT_COMPILATION_MODE = wholemodule;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
||||
VALIDATE_PRODUCT = YES;
|
||||
VERSIONING_SYSTEM = "apple-generic";
|
||||
VERSION_INFO_PREFIX = "";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
8566B5612ABABF9A00AAB22A /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEFINES_MODULE = YES;
|
||||
DYLIB_COMPATIBILITY_VERSION = 1;
|
||||
DYLIB_CURRENT_VERSION = 1;
|
||||
DYLIB_INSTALL_NAME_BASE = "@rpath";
|
||||
ENABLE_MODULE_VERIFIER = YES;
|
||||
GENERATE_INFOPLIST_FILE = YES;
|
||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
||||
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 1.0;
|
||||
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
|
||||
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
|
||||
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
|
||||
SKIP_INSTALL = YES;
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_VERSION = 5.0;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8566B5622ABABF9A00AAB22A /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEFINES_MODULE = YES;
|
||||
DYLIB_COMPATIBILITY_VERSION = 1;
|
||||
DYLIB_CURRENT_VERSION = 1;
|
||||
DYLIB_INSTALL_NAME_BASE = "@rpath";
|
||||
ENABLE_MODULE_VERIFIER = YES;
|
||||
GENERATE_INFOPLIST_FILE = YES;
|
||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
||||
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 1.0;
|
||||
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
|
||||
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
|
||||
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
|
||||
SKIP_INSTALL = YES;
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_VERSION = 5.0;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8566B55E2ABABF9A00AAB22A /* Debug */,
|
||||
8566B55F2ABABF9A00AAB22A /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8566B5612ABABF9A00AAB22A /* Debug */,
|
||||
8566B5622ABABF9A00AAB22A /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
/* End XCConfigurationList section */
|
||||
};
|
||||
rootObject = 8566B5502ABABF9A00AAB22A /* Project object */;
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Workspace
|
||||
version = "1.0">
|
||||
<FileRef
|
||||
location = "self:">
|
||||
</FileRef>
|
||||
</Workspace>
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>IDEDidComputeMac32BitWarning</key>
|
||||
<true/>
|
||||
</dict>
|
||||
</plist>
|
Binary file not shown.
|
@ -0,0 +1,14 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>SchemeUserState</key>
|
||||
<dict>
|
||||
<key>MediaPipeTasksDocGen.xcscheme_^#shared#^_</key>
|
||||
<dict>
|
||||
<key>orderHint</key>
|
||||
<integer>0</integer>
|
||||
</dict>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
|
@ -0,0 +1,17 @@
|
|||
//
|
||||
// MediaPipeTasksDocGen.h
|
||||
// MediaPipeTasksDocGen
|
||||
//
|
||||
// Created by Mark McDonald on 20/9/2023.
|
||||
//
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
//! Project version number for MediaPipeTasksDocGen.
|
||||
FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber;
|
||||
|
||||
//! Project version string for MediaPipeTasksDocGen.
|
||||
FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[];
|
||||
|
||||
// In this header, you should import all the public headers of your framework using statements like
|
||||
// #import <MediaPipeTasksDocGen/PublicHeader.h>
|
11
docs/MediaPipeTasksDocGen/Podfile
Normal file
11
docs/MediaPipeTasksDocGen/Podfile
Normal file
|
@ -0,0 +1,11 @@
|
|||
# Uncomment the next line to define a global platform for your project
|
||||
platform :ios, '15.0'
|
||||
|
||||
target 'MediaPipeTasksDocGen' do
|
||||
# Comment the next line if you don't want to use dynamic frameworks
|
||||
use_frameworks!
|
||||
|
||||
# Pods for MediaPipeTasksDocGen
|
||||
pod 'MediaPipeTasksText'
|
||||
pod 'MediaPipeTasksVision'
|
||||
end
|
9
docs/MediaPipeTasksDocGen/README.md
Normal file
9
docs/MediaPipeTasksDocGen/README.md
Normal file
|
@ -0,0 +1,9 @@
|
|||
# MediaPipeTasksDocGen
|
||||
|
||||
This empty project is used to generate reference documentation for the
|
||||
ObjectiveC and Swift libraries.
|
||||
|
||||
Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published
|
||||
to [the developer site](https://developers.google.com/mediapipe/solutions/).
|
||||
|
||||
To bump the API version used, edit [`Podfile`](./Podfile).
|
|
@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
|
|||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// the cumulative timestamping, which is inferred from the initial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 8 [default = false];
|
||||
}
|
||||
|
|
|
@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
|
|||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// the cumulative timestamping, which is inferred from the initial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 6 [default = false];
|
||||
}
|
||||
|
|
|
@ -727,6 +727,7 @@ cc_library(
|
|||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
@ -742,6 +743,7 @@ cc_test(
|
|||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:options_util",
|
||||
"//mediapipe/util:packet_test_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
|
|
|
@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
|
|||
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
|
||||
}
|
||||
|
||||
// Shows how control packets recieve timestamps before and after frame packets
|
||||
// Shows how control packets receive timestamps before and after frame packets
|
||||
// have arrived.
|
||||
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
|
||||
CalculatorGraphConfig::Node node_config = BuildNodeConfig();
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
@ -32,6 +33,7 @@ namespace {
|
|||
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
|
||||
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
|
||||
constexpr char kTagAtZero[] = "AT_ZERO";
|
||||
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
|
||||
constexpr char kTagAtTick[] = "AT_TICK";
|
||||
constexpr char kTagTick[] = "TICK";
|
||||
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
|
||||
|
@ -43,6 +45,7 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
|
|||
res->emplace(kTagAtPostStream, Timestamp::PostStream());
|
||||
res->emplace(kTagAtZero, Timestamp(0));
|
||||
res->emplace(kTagAtTick, Timestamp::Unset());
|
||||
res->emplace(kTagAtFirstTick, Timestamp::Unset());
|
||||
res->emplace(kTagAtTimestamp, Timestamp::Unset());
|
||||
return res;
|
||||
}();
|
||||
|
@ -59,8 +62,8 @@ std::string GetOutputTag(const CC& cc) {
|
|||
// timestamp, depending on the tag used to define output stream(s). (One tag can
|
||||
// be used only.)
|
||||
//
|
||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
|
||||
// and corresponding timestamps are Timestamp::PreStream(),
|
||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
|
||||
// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
|
||||
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
|
||||
// input, and timestamp received from a side input.
|
||||
//
|
||||
|
@ -96,6 +99,7 @@ class SidePacketToStreamCalculator : public CalculatorBase {
|
|||
|
||||
private:
|
||||
bool is_tick_processing_ = false;
|
||||
bool close_on_first_tick_ = false;
|
||||
std::string output_tag_;
|
||||
};
|
||||
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||
|
@ -103,13 +107,16 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
|||
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
||||
const auto& tags = cc->Outputs().GetTags();
|
||||
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
|
||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s).";
|
||||
RET_CHECK(
|
||||
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
|
||||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
|
||||
<< "Either both of TICK and AT_TICK should be used or none of them.";
|
||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
|
||||
"AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
|
||||
"specify output stream(s).";
|
||||
const bool has_tick_output =
|
||||
cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
|
||||
const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
|
||||
RET_CHECK((has_tick_output && has_tick_input) ||
|
||||
(!has_tick_output && !has_tick_input))
|
||||
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them.";
|
||||
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
|
||||
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||
|
@ -148,11 +155,17 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
|
|||
// timestamp bound update.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
}
|
||||
if (output_tag_ == kTagAtFirstTick) {
|
||||
close_on_first_tick_ = true;
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
||||
if (is_tick_processing_) {
|
||||
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// TICK input is guaranteed to be non-empty, as it's the only input stream
|
||||
// for this calculator.
|
||||
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
|
||||
|
@ -160,6 +173,9 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
|||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
if (close_on_first_tick_) {
|
||||
cc->Outputs().Get(output_tag_, i).Close();
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
|
@ -170,6 +186,7 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
|||
|
||||
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
if (!cc->Outputs().HasTag(kTagAtTick) &&
|
||||
!cc->Outputs().HasTag(kTagAtFirstTick) &&
|
||||
!cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
const auto& timestamp = kTimestampMap->at(output_tag_);
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
|
|
|
@ -27,13 +27,17 @@
|
|||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/options_util.h"
|
||||
#include "mediapipe/util/packet_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using testing::HasSubstr;
|
||||
using ::testing::ElementsAre;
|
||||
using ::testing::Eq;
|
||||
using ::testing::HasSubstr;
|
||||
using ::testing::IsEmpty;
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -52,10 +56,35 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
|||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr(
|
||||
"Either both of TICK and AT_TICK should be used or none of them."));
|
||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
TEST(SidePacketToStreamCalculator,
|
||||
WrongConfigWithMissingTickForFirstTickProcessing) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_FIRST_TICK:packet"
|
||||
}
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr(
|
||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -76,7 +105,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
|||
"or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -92,14 +121,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
|||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s)."));
|
||||
EXPECT_THAT(status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
||||
"allowed and required to specify output stream(s)."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -117,14 +145,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
|||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s)."));
|
||||
EXPECT_THAT(status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
||||
"allowed and required to specify output stream(s)."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -146,7 +173,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
|||
"Same number of input side packets and output streams is required."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -248,7 +275,50 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
|||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, AtFirstTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_FIRST_TICK:packet"
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value = 20;
|
||||
const Timestamp kTestTimestamp(1234);
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
|
||||
|
||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
};
|
||||
|
||||
insert_tick(kTestTimestamp);
|
||||
|
||||
EXPECT_THAT(output_packets,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value))));
|
||||
|
||||
output_packets.clear();
|
||||
|
||||
// Should not result in an additional output.
|
||||
insert_tick(kTestTimestamp + 1);
|
||||
EXPECT_THAT(output_packets, IsEmpty());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -302,6 +372,62 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
|||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "packet0"
|
||||
output_stream: "packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_FIRST_TICK:0:packet0"
|
||||
output_stream: "AT_FIRST_TICK:1:packet1"
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value0 = 20;
|
||||
const int expected_value1 = 128;
|
||||
const Timestamp kTestTimestamp(1234);
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
|
||||
{"side_packet1", MakePacket<int>(expected_value1)}}));
|
||||
|
||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
};
|
||||
|
||||
insert_tick(kTestTimestamp);
|
||||
|
||||
EXPECT_THAT(output_packets0,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value0))));
|
||||
EXPECT_THAT(output_packets1,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value1))));
|
||||
|
||||
output_packets0.clear();
|
||||
output_packets1.clear();
|
||||
|
||||
// Should not result in an additional output.
|
||||
insert_tick(kTestTimestamp + 1);
|
||||
EXPECT_THAT(output_packets0, IsEmpty());
|
||||
EXPECT_THAT(output_packets1, IsEmpty());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
|
@ -334,7 +460,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
|||
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
|
|
@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
|
|||
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
|
||||
}
|
||||
|
||||
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
|
||||
TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
|
||||
// Check that when we provide the inputs not on time - we don't get them.
|
||||
ValueOrDefaultRunner runner;
|
||||
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
||||
|
|
|
@ -59,7 +59,7 @@ class OpenCvRunner
|
|||
const ImageFrame& input, const std::array<float, 16>& matrix,
|
||||
const AffineTransformation::Size& size,
|
||||
AffineTransformation::BorderMode border_mode) override {
|
||||
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
|
||||
// OpenCV warpAffine works in absolute coordinates, so the transform (which
|
||||
// accepts and produces relative coordinates) should be adjusted to first
|
||||
// normalize coordinates and then scale them.
|
||||
// clang-format off
|
||||
|
|
|
@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
|
|||
}
|
||||
#else
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
|
|
@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
|
|||
}
|
||||
|
||||
// Output texture buffer dimensions. The values defined in the options will be
|
||||
// overriden by the WIDTH and HEIGHT input streams if they exist.
|
||||
// overridden by the WIDTH and HEIGHT input streams if they exist.
|
||||
optional int32 width = 1;
|
||||
optional int32 height = 2;
|
||||
|
||||
|
|
|
@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
|
|||
return focal_length_pixels;
|
||||
}
|
||||
|
||||
absl::StatusOr<ImageFileProperties> GetImageFileProperites(
|
||||
absl::StatusOr<ImageFileProperties> GetImageFileProperties(
|
||||
const std::string& image_bytes) {
|
||||
easyexif::EXIFInfo result;
|
||||
int code = result.parseFrom(image_bytes);
|
||||
|
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||
if (cc->InputSidePackets().NumEntries() == 1) {
|
||||
const std::string& image_bytes =
|
||||
cc->InputSidePackets().Index(0).Get<std::string>();
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
|
||||
|
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||
return absl::OkStatus();
|
||||
}
|
||||
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
if (read_properties_) {
|
||||
|
|
|
@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
|
|||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
return absl::OkStatus();
|
||||
|
|
|
@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
|||
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
|
||||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
|
|
@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
|
|||
.IgnoreError();
|
||||
mediapipe::GetRotatedSubRectToRectTransformMatrix(
|
||||
roi_absolute, input.cols, input.rows,
|
||||
/*flip_horizontaly=*/false, &transform_mat);
|
||||
/*flip_horizontally=*/false, &transform_mat);
|
||||
return transform_mat;
|
||||
}
|
||||
|
||||
|
|
|
@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
|
|||
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
|
||||
// YV21) format (as per the `fourcc()` property). This covers the most commonly
|
||||
// used YUV image formats used on mobile devices. Other formats are not
|
||||
// supported and wil result in an `InvalidArgumentError`.
|
||||
// supported and will result in an `InvalidArgumentError`.
|
||||
class YUVToImageCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
|
||||
|
|
|
@ -657,6 +657,7 @@ cc_library(
|
|||
}),
|
||||
deps = [
|
||||
":tensor_converter_calculator_cc_proto",
|
||||
":tensor_converter_cpu",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
|
@ -665,6 +666,7 @@ cc_library(
|
|||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
"//mediapipe/util:resource_util",
|
||||
|
@ -674,10 +676,17 @@ cc_library(
|
|||
"@com_google_absl//absl/log:check",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
"//conditions:default": [
|
||||
"tensor_converter_calculator_gpu_deps",
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
}) + select({
|
||||
"//mediapipe:apple": [
|
||||
"//third_party/apple_frameworks:MetalKit",
|
||||
|
@ -687,6 +696,35 @@ cc_library(
|
|||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_cpu",
|
||||
srcs = ["tensor_converter_cpu.cc"],
|
||||
hdrs = ["tensor_converter_cpu.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_converter_cpu_test",
|
||||
srcs = ["tensor_converter_cpu_test.cc"],
|
||||
deps = [
|
||||
":tensor_converter_cpu",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status_matchers",
|
||||
"//mediapipe/util:image_test_utils",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator_gpu_deps",
|
||||
visibility = ["//visibility:private"],
|
||||
|
@ -1414,6 +1452,8 @@ cc_library(
|
|||
}),
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_converter",
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
|
@ -1421,9 +1461,11 @@ cc_library(
|
|||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
"//mediapipe/util:resource_util",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
|
@ -1434,6 +1476,7 @@ cc_library(
|
|||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
}) + selects.with_or({
|
||||
|
@ -1453,19 +1496,96 @@ cc_library(
|
|||
}) + select({
|
||||
"//mediapipe/framework/port:disable_opencv": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/framework/formats:image_opencv",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
":tensors_to_segmentation_converter_opencv",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_utils",
|
||||
srcs = ["tensors_to_segmentation_utils.cc"],
|
||||
hdrs = ["tensors_to_segmentation_utils.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_utils_test",
|
||||
srcs = ["tensors_to_segmentation_utils_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status_matchers",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_converter",
|
||||
hdrs = ["tensors_to_segmentation_converter.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_converter_opencv",
|
||||
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
|
||||
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_converter",
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_opencv",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_calculator_test_utils",
|
||||
testonly = 1,
|
||||
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
|
||||
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_calculator_test_utils_test",
|
||||
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_calculator_test_utils",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_calculator_test",
|
||||
srcs = ["tensors_to_segmentation_calculator_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator",
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_calculator_test_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
|
@ -1476,11 +1596,6 @@ cc_test(
|
|||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/log",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_googletest//:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
|
|||
// Non-streaming mode: when "stream_mode" is set to false in the calculator
|
||||
// options, the calculators treats the packets in the input audio stream as
|
||||
// a batch of unrelated audio buffers. In each Process() call, the input
|
||||
// buffer will be frist resampled, and framed as fixed-sized, possibly
|
||||
// buffer will be first resampled, and framed as fixed-sized, possibly
|
||||
// overlapping tensors. The last tensor produced by a Process() invocation
|
||||
// will be zero-padding if the remaining samples are insufficient. As the
|
||||
// calculator treats the input packets as unrelated, all samples will be
|
||||
|
@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
|
|||
public:
|
||||
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
|
||||
// TODO: Removes this optional input stream when the "AUDIO" stream
|
||||
// uses the new mediapipe audio data containers that carry audio metatdata,
|
||||
// uses the new mediapipe audio data containers that carry audio metadata,
|
||||
// such as sample rate.
|
||||
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
|
||||
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
|
||||
|
|
|
@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
|
|||
// will be converted into tensors.
|
||||
optional double target_sample_rate = 4;
|
||||
|
||||
// Whether to treat the input audio stream as a continous stream or a batch
|
||||
// Whether to treat the input audio stream as a continuous stream or a batch
|
||||
// of unrelated audio buffers.
|
||||
optional bool stream_mode = 5 [default = true];
|
||||
|
||||
|
|
|
@ -82,7 +82,7 @@ namespace api2 {
|
|||
//
|
||||
// Outputs:
|
||||
// TENSORS - std::vector<Tensor>
|
||||
// Vector containing a single Tensor populated with an extrated RGB image.
|
||||
// Vector containing a single Tensor populated with an extracted RGB image.
|
||||
// MATRIX - std::array<float, 16> @Optional
|
||||
// An std::array<float, 16> representing a 4x4 row-major-order matrix that
|
||||
// maps a point on the input image to a point on the output tensor, and
|
||||
|
@ -212,7 +212,7 @@ class ImageToTensorCalculator : public Node {
|
|||
std::array<float, 16> matrix;
|
||||
GetRotatedSubRectToRectTransformMatrix(
|
||||
roi, image->width(), image->height(),
|
||||
/*flip_horizontaly=*/false, &matrix);
|
||||
/*flip_horizontally=*/false, &matrix);
|
||||
kOutMatrix(cc).Send(std::move(matrix));
|
||||
}
|
||||
|
||||
|
|
|
@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
|
|||
} else if (image_channels == 1) {
|
||||
return ImageFormat::GRAY8;
|
||||
}
|
||||
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
|
||||
ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
|
||||
}
|
||||
|
||||
Packet MakeImageFramePacket(cv::Mat input) {
|
||||
|
|
|
@ -57,7 +57,7 @@ class SubRectExtractorGl {
|
|||
absl::Status ExtractSubRectToBuffer(
|
||||
const tflite::gpu::gl::GlTexture& texture,
|
||||
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
bool flip_horizontally, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
tflite::gpu::gl::CommandQueue* command_queue,
|
||||
tflite::gpu::gl::GlBuffer* destination);
|
||||
|
@ -154,13 +154,13 @@ void main() {
|
|||
absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
|
||||
const tflite::gpu::gl::GlTexture& texture,
|
||||
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
bool flip_horizontally, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
tflite::gpu::gl::CommandQueue* command_queue,
|
||||
tflite::gpu::gl::GlBuffer* destination) {
|
||||
std::array<float, 16> transform_mat;
|
||||
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
|
||||
texture_size.h, flip_horizontaly,
|
||||
texture_size.h, flip_horizontally,
|
||||
&transform_mat);
|
||||
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
|
||||
|
||||
|
@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter {
|
|||
input_texture,
|
||||
tflite::gpu::HW(source_texture.height(), source_texture.width()),
|
||||
roi,
|
||||
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||
/*flip_horizontally=*/false, transform.scale, transform.offset,
|
||||
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
||||
command_queue_.get(), &output));
|
||||
|
||||
|
|
|
@ -199,7 +199,7 @@ class GlProcessor : public ImageToTensorConverter {
|
|||
range_min, range_max));
|
||||
auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
|
||||
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
|
||||
/*flip_horizontaly=*/false,
|
||||
/*flip_horizontally=*/false,
|
||||
transform.scale, transform.offset,
|
||||
output_shape, &tensor_view));
|
||||
return absl::OkStatus();
|
||||
|
@ -210,7 +210,7 @@ class GlProcessor : public ImageToTensorConverter {
|
|||
|
||||
absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
|
||||
const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
bool flip_horizontally, float alpha, float beta,
|
||||
const Tensor::Shape& output_shape,
|
||||
Tensor::OpenGlTexture2dView* output) {
|
||||
const int output_height = output_shape.dims[1];
|
||||
|
@ -263,13 +263,13 @@ class GlProcessor : public ImageToTensorConverter {
|
|||
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
|
||||
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
|
||||
GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||
sub_rect, texture.width(), texture.height(), flip_horizontaly,
|
||||
sub_rect, texture.width(), texture.height(), flip_horizontally,
|
||||
&transform_mat);
|
||||
glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
|
||||
} else {
|
||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
|
||||
texture.height(), flip_horizontaly,
|
||||
&transform_mat);
|
||||
texture.height(),
|
||||
flip_horizontally, &transform_mat);
|
||||
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
|
||||
}
|
||||
|
||||
|
|
|
@ -179,13 +179,13 @@ class SubRectExtractorMetal {
|
|||
}
|
||||
|
||||
absl::Status Execute(id<MTLTexture> input_texture,
|
||||
const RotatedRect& sub_rect, bool flip_horizontaly,
|
||||
const RotatedRect& sub_rect, bool flip_horizontally,
|
||||
float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
id<MTLCommandBuffer> command_buffer,
|
||||
id<MTLBuffer> destination) {
|
||||
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
|
||||
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
|
||||
return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
|
||||
beta, destination_size, command_buffer,
|
||||
output_texture);
|
||||
}
|
||||
|
@ -211,7 +211,7 @@ class SubRectExtractorMetal {
|
|||
|
||||
absl::Status InternalExecute(id<MTLTexture> input_texture,
|
||||
const RotatedRect& sub_rect,
|
||||
bool flip_horizontaly, float alpha, float beta,
|
||||
bool flip_horizontally, float alpha, float beta,
|
||||
const tflite::gpu::HW& destination_size,
|
||||
id<MTLCommandBuffer> command_buffer,
|
||||
id<MTLTexture> output_texture) {
|
||||
|
@ -223,7 +223,7 @@ class SubRectExtractorMetal {
|
|||
std::array<float, 16> transform_mat;
|
||||
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
|
||||
input_texture.height,
|
||||
flip_horizontaly, &transform_mat);
|
||||
flip_horizontally, &transform_mat);
|
||||
id<MTLBuffer> transform_mat_buffer =
|
||||
[device_ newBufferWithBytes:&transform_mat
|
||||
length:sizeof(transform_mat)
|
||||
|
@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
|
|||
MtlBufferView::GetWriteView(output_tensor, command_buffer);
|
||||
MP_RETURN_IF_ERROR(extractor_->Execute(
|
||||
texture, roi,
|
||||
/*flip_horizontaly=*/false, transform.scale, transform.offset,
|
||||
/*flip_horizontally=*/false, transform.scale, transform.offset,
|
||||
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
|
||||
command_buffer, buffer_view.buffer()));
|
||||
[command_buffer commit];
|
||||
|
|
|
@ -92,7 +92,7 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
|||
|
||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||
int rect_width, int rect_height,
|
||||
bool flip_horizontaly,
|
||||
bool flip_horizontally,
|
||||
std::array<float, 16>* matrix_ptr) {
|
||||
std::array<float, 16>& matrix = *matrix_ptr;
|
||||
// The resulting matrix is multiplication of below commented out matrices:
|
||||
|
@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
|||
// {0.0f, 0.0f, a, 0.0f}
|
||||
// {0.0f, 0.0f, 0.0f, 1.0f}
|
||||
|
||||
const float flip = flip_horizontaly ? -1 : 1;
|
||||
const float flip = flip_horizontally ? -1 : 1;
|
||||
// Matrix for optional horizontal flip around middle of output image.
|
||||
// { fl , 0.0f, 0.0f, 0.0f}
|
||||
// { 0.0f, 1.0f, 0.0f, 0.0f}
|
||||
|
@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
|||
|
||||
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
||||
bool flip_horizontaly, std::array<float, 16>* matrix_ptr) {
|
||||
bool flip_horizontally, std::array<float, 16>* matrix_ptr) {
|
||||
std::array<float, 16>& matrix = *matrix_ptr;
|
||||
// See comments in GetRotatedSubRectToRectTransformMatrix for detailed
|
||||
// calculations.
|
||||
const float a = sub_rect.width;
|
||||
const float b = sub_rect.height;
|
||||
const float flip = flip_horizontaly ? -1 : 1;
|
||||
const float flip = flip_horizontally ? -1 : 1;
|
||||
const float c = std::cos(sub_rect.rotation);
|
||||
const float d = std::sin(sub_rect.rotation);
|
||||
const float e = sub_rect.center_x;
|
||||
|
|
|
@ -74,7 +74,7 @@ absl::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
|
|||
// Represents a transformation of value which involves scaling and offsetting.
|
||||
// To apply transformation:
|
||||
// ValueTransformation transform = ...
|
||||
// float transformed_value = transform.scale * value + transfrom.offset;
|
||||
// float transformed_value = transform.scale * value + transform.offset;
|
||||
struct ValueTransformation {
|
||||
float scale;
|
||||
float offset;
|
||||
|
@ -99,11 +99,11 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
|
|||
// @sub_rect - rotated sub rect in absolute coordinates
|
||||
// @rect_width - rect width
|
||||
// @rect_height - rect height
|
||||
// @flip_horizontaly - we need to flip the output buffer.
|
||||
// @flip_horizontally - we need to flip the output buffer.
|
||||
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
||||
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
||||
int rect_width, int rect_height,
|
||||
bool flip_horizontaly,
|
||||
bool flip_horizontally,
|
||||
std::array<float, 16>* matrix);
|
||||
|
||||
// Returns the transpose of the matrix found with
|
||||
|
@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
|
|||
// @sub_rect - rotated sub rect in absolute coordinates
|
||||
// @rect_width - rect width
|
||||
// @rect_height - rect height
|
||||
// @flip_horizontaly - we need to flip the output buffer.
|
||||
// @flip_horizontally - we need to flip the output buffer.
|
||||
// @matrix - 4x4 matrix (array of 16 elements) to populate
|
||||
void GetTransposedRotatedSubRectToRectTransformMatrix(
|
||||
const RotatedRect& sub_rect, int rect_width, int rect_height,
|
||||
bool flip_horizontaly, std::array<float, 16>* matrix);
|
||||
bool flip_horizontally, std::array<float, 16>* matrix);
|
||||
|
||||
// Validates the output dimensions set in the option proto. The input option
|
||||
// proto is expected to have to following fields:
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
|
@ -21,17 +22,22 @@
|
|||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/gpu/gpu_buffer_format.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
@ -94,16 +100,13 @@ absl::StatusOr<bool> ShouldFlipVertically(
|
|||
}
|
||||
}
|
||||
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
|
||||
RowMajorMatrixXf;
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
|
||||
ColMajorMatrixXf;
|
||||
|
||||
constexpr char kImageFrameTag[] = "IMAGE";
|
||||
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
constexpr char kMatrixTag[] = "MATRIX";
|
||||
|
||||
constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
@ -156,10 +159,6 @@ class TensorConverterCalculator : public CalculatorBase {
|
|||
private:
|
||||
absl::Status InitGpu(CalculatorContext* cc);
|
||||
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
|
||||
template <class T>
|
||||
absl::Status NormalizeImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically, float* tensor_ptr);
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
|
||||
absl::Status ProcessCPU(CalculatorContext* cc);
|
||||
absl::Status ProcessGPU(CalculatorContext* cc);
|
||||
|
||||
|
@ -279,46 +278,21 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
|
|||
}
|
||||
const auto& image_frame =
|
||||
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const mediapipe::ImageFormat::Format format = image_frame.Format();
|
||||
|
||||
if (!(format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::VEC32F1))
|
||||
RET_CHECK_FAIL() << "Unsupported CPU input format.";
|
||||
|
||||
output_tensors->emplace_back(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels_preserved});
|
||||
auto cpu_view = output_tensors->back().GetCpuWriteView();
|
||||
|
||||
// Copy image data into tensor.
|
||||
if (image_frame.ByteDepth() == 1) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else if (image_frame.ByteDepth() == 4) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else {
|
||||
return absl::InternalError(
|
||||
"Only byte-based (8 bit) and float (32 bit) images supported.");
|
||||
}
|
||||
MP_ASSIGN_OR_RETURN(Tensor output,
|
||||
ConvertImageFrameToTensorOnCpu(
|
||||
image_frame,
|
||||
output_range_.has_value() ? output_range_.value()
|
||||
: kDefaultOutputRange,
|
||||
flip_vertically_, max_num_channels_));
|
||||
output_tensors->emplace_back(std::move(output));
|
||||
} else if (cc->Inputs().HasTag(kMatrixTag)) {
|
||||
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
|
||||
const int height = matrix.rows();
|
||||
const int width = matrix.cols();
|
||||
const int channels = 1;
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
|
||||
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
|
||||
MP_ASSIGN_OR_RETURN(Tensor output,
|
||||
ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
|
||||
output_tensors->emplace_back(std::move(output));
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
@ -669,67 +643,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
|
|||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
template <class T>
|
||||
absl::Status TensorConverterCalculator::NormalizeImage(
|
||||
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const int channels_ignored = channels - channels_preserved;
|
||||
|
||||
if (output_range_.has_value()) {
|
||||
// If the output float range is set and we are not using custom
|
||||
// normalization, normalize the pixel values from [0, 255] to the specified
|
||||
// output range.
|
||||
RET_CHECK_NE(output_range_->first, output_range_->second);
|
||||
const float scale = (output_range_->second - output_range_->first) / 255.0f;
|
||||
const float bias = output_range_->first;
|
||||
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale + bias;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// [0,1], scale only (bias == 0)
|
||||
// Verified that there are no precision issues with 1.0f / 255.0f expression
|
||||
const float scale = 1.0f / 255.0f;
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
|
||||
float* tensor_ptr) {
|
||||
if (row_major_matrix_) {
|
||||
auto matrix_map =
|
||||
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
} else {
|
||||
auto matrix_map =
|
||||
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -32,7 +32,7 @@ message TensorConverterCalculatorOptions {
|
|||
// Custom settings to override the internal scaling factors `div` and `sub`.
|
||||
// Both values must be set to non-negative values. Will only take effect on
|
||||
// CPU AND when |use_custom_normalization| is set to true. When these custom
|
||||
// values take effect, the |zero_center| setting above will be overriden, and
|
||||
// values take effect, the |zero_center| setting above will be overridden, and
|
||||
// the normalized_value will be calculated as:
|
||||
// normalized_value = input / custom_div - custom_sub.
|
||||
optional bool use_custom_normalization = 6 [default = false];
|
||||
|
|
|
@ -321,6 +321,61 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest,
|
||||
ShouldConvertImageWithDefaultOutputRange) {
|
||||
CalculatorGraph graph;
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
zero_center: false
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
|
||||
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
|
||||
mat.at<uint8_t>(0, 0) = 200;
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
|
||||
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_EQ(output_packets.size(), 1);
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
ASSERT_EQ(tensor_vec.size(), 1);
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
|
||||
// Calculate the expected normalized value:
|
||||
float expected_value = 200.0 / 255.0;
|
||||
|
||||
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
|
||||
auto view = tensor->GetCpuReadView();
|
||||
float actual_value = *view.buffer<float>();
|
||||
EXPECT_FLOAT_EQ(actual_value, expected_value);
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, FlipVertically) {
|
||||
CalculatorGraph graph;
|
||||
CalculatorGraphConfig graph_config =
|
||||
|
|
145
mediapipe/calculators/tensor/tensor_converter_cpu.cc
Normal file
145
mediapipe/calculators/tensor/tensor_converter_cpu.cc
Normal file
|
@ -0,0 +1,145 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
|
||||
RowMajorMatrixXf;
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
|
||||
ColMajorMatrixXf;
|
||||
|
||||
template <class T>
|
||||
absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels);
|
||||
const int channels_ignored = channels - channels_preserved;
|
||||
|
||||
RET_CHECK_NE(output_range.first, output_range.second);
|
||||
const float scale = (output_range.second - output_range.first) / 255.0f;
|
||||
const float bias = output_range.first;
|
||||
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale + bias;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
|
||||
max_num_channels, tensor_ptr);
|
||||
}
|
||||
|
||||
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
return NormalizeImage<float>(image_frame, flip_vertically, output_range,
|
||||
max_num_channels, tensor_ptr);
|
||||
}
|
||||
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
|
||||
float* tensor_ptr) {
|
||||
if (is_row_major_matrix) {
|
||||
auto matrix_map =
|
||||
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
} else {
|
||||
auto matrix_map =
|
||||
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
|
||||
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
|
||||
bool flip_vertically, int max_num_channels) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels);
|
||||
const mediapipe::ImageFormat::Format format = image_frame.Format();
|
||||
|
||||
if (!(format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::VEC32F1))
|
||||
RET_CHECK_FAIL() << "Unsupported CPU input format.";
|
||||
|
||||
Tensor output_tensor(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels_preserved});
|
||||
auto cpu_view = output_tensor.GetCpuWriteView();
|
||||
|
||||
// Copy image data into tensor.
|
||||
if (image_frame.ByteDepth() == 1) {
|
||||
MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
|
||||
output_range, max_num_channels,
|
||||
cpu_view.buffer<float>()));
|
||||
} else if (image_frame.ByteDepth() == 4) {
|
||||
MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
|
||||
output_range, max_num_channels,
|
||||
cpu_view.buffer<float>()));
|
||||
} else {
|
||||
return absl::InternalError(
|
||||
"Only byte-based (8 bit) and float (32 bit) images supported.");
|
||||
}
|
||||
return output_tensor;
|
||||
}
|
||||
|
||||
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
|
||||
bool row_major_matrix) {
|
||||
const int height = matrix.rows();
|
||||
const int width = matrix.cols();
|
||||
const int channels = 1;
|
||||
Tensor output_tensor(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
MP_RETURN_IF_ERROR(
|
||||
CopyMatrixToTensor(matrix, row_major_matrix,
|
||||
output_tensor.GetCpuWriteView().buffer<float>()));
|
||||
return output_tensor;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
61
mediapipe/calculators/tensor/tensor_converter_cpu.h
Normal file
61
mediapipe/calculators/tensor/tensor_converter_cpu.h
Normal file
|
@ -0,0 +1,61 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Converts an ImageFrame to a vector of Tensors.
|
||||
// @flip_vertically enables to flip the image during conversion.
|
||||
// @max_num_channels can be used to reserve extra channels in the output
|
||||
// tensors.
|
||||
// Returns output Tensor.
|
||||
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
|
||||
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
|
||||
bool flip_vertically, int max_num_channels);
|
||||
|
||||
// Converts a Matrix to a vector of Tensors.
|
||||
// @row_major_matrix defines the ordering in the input matrix.
|
||||
// @max_num_channels can be used to reserve extra channels in the output
|
||||
// tensors.
|
||||
// Returns output Tensor.
|
||||
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
|
||||
bool row_major_matrix);
|
||||
|
||||
// For testing only below.
|
||||
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr);
|
||||
|
||||
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr);
|
||||
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
|
||||
float* tensor_ptr);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
175
mediapipe/calculators/tensor/tensor_converter_cpu_test.cc
Normal file
175
mediapipe/calculators/tensor/tensor_converter_cpu_test.cc
Normal file
|
@ -0,0 +1,175 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/util/image_test_utils.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
Matrix CreateTestMatrix(int num_rows, int num_columns) {
|
||||
Matrix matrix(num_rows, num_columns);
|
||||
for (int r = 0; r < num_rows; ++r) {
|
||||
for (int c = 0; c < num_columns; ++c) {
|
||||
matrix(r, c) = r * num_columns + c;
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
|
||||
auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
|
||||
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int row = i / test_matrix.cols();
|
||||
const int column = i % test_matrix.cols();
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
|
||||
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
|
||||
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int row = i % test_matrix.rows();
|
||||
const int column = i / test_matrix.rows();
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_data[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
const auto range = std::make_pair(2.0f, 3.0f);
|
||||
|
||||
MP_EXPECT_OK(
|
||||
NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
|
||||
/*num_tensor_channels=*/1, tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_data[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
|
||||
255.0f * (range.second - range.first) +
|
||||
range.first);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int x = i % grey8_image_frame.Width();
|
||||
const int y = i / grey8_image_frame.Width();
|
||||
const int flipped_y = grey8_image_frame.Height() - y - 1;
|
||||
|
||||
const int index = flipped_y * grey8_image_frame.Width() + x;
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_data[index],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
|
||||
auto float_image_frame =
|
||||
CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
float_image_frame.Width() * float_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
|
||||
float_image_frame.PixelData())[i] /
|
||||
255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
|
||||
MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
|
||||
grey8_image_frame, {0.0f, 1.0f},
|
||||
/*flip_vertically=*/false,
|
||||
/*max_num_channels=*/1));
|
||||
|
||||
const auto cpu_read_view = output.GetCpuReadView();
|
||||
const float* tensor_ptr = cpu_read_view.buffer<float>();
|
||||
for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
|
||||
++i) {
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_ptr[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
|
||||
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
|
||||
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
|
||||
/*row_major_matrix=*/false));
|
||||
|
||||
const auto cpu_read_view = output.GetCpuReadView();
|
||||
const float* tensor_ptr = cpu_read_view.buffer<float>();
|
||||
for (int i = 0; i < test_matrix.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
|
@ -34,7 +34,7 @@ message TensorsToClassificationCalculatorOptions {
|
|||
repeated Entry entries = 1;
|
||||
}
|
||||
|
||||
// Score threshold for perserving the class.
|
||||
// Score threshold for preserving the class.
|
||||
optional float min_score_threshold = 1;
|
||||
// Number of highest scoring labels to output. If top_k is not positive then
|
||||
// all labels are used.
|
||||
|
|
|
@ -15,7 +15,6 @@
|
|||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
|
||||
|
@ -147,7 +146,7 @@ BoxFormat GetBoxFormat(const TensorsToDetectionsCalculatorOptions& options) {
|
|||
// TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have
|
||||
// 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints.
|
||||
// The size of the values must be (num_boxes * num_predicted_values).
|
||||
// Second tensor is the score tensor. The size of the valuse must be
|
||||
// Second tensor is the score tensor. The size of the values must be
|
||||
// (num_boxes * num_classes). It's optional to pass in a third tensor
|
||||
// for anchors (e.g. for SSD models) depend on the outputs of the
|
||||
// detection model. The size of anchor tensor must be (num_boxes *
|
||||
|
@ -215,7 +214,8 @@ class TensorsToDetectionsCalculator : public Node {
|
|||
const int* detection_classes,
|
||||
std::vector<Detection>* output_detections);
|
||||
Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
|
||||
float box_xmax, float score, int class_id,
|
||||
float box_xmax, absl::Span<const float> scores,
|
||||
absl::Span<const int> class_ids,
|
||||
bool flip_vertically);
|
||||
bool IsClassIndexAllowed(int class_index);
|
||||
|
||||
|
@ -223,6 +223,7 @@ class TensorsToDetectionsCalculator : public Node {
|
|||
int num_boxes_ = 0;
|
||||
int num_coords_ = 0;
|
||||
int max_results_ = -1;
|
||||
int classes_per_detection_ = 1;
|
||||
BoxFormat box_output_format_ =
|
||||
mediapipe::TensorsToDetectionsCalculatorOptions::YXHW;
|
||||
|
||||
|
@ -267,7 +268,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract(
|
|||
if (CanUseGpu()) {
|
||||
#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
#elif MEDIAPIPE_METAL_ENABLED
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
|
||||
|
@ -484,6 +485,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
|
|||
auto num_boxes_view = num_boxes_tensor->GetCpuReadView();
|
||||
auto num_boxes = num_boxes_view.buffer<float>();
|
||||
num_boxes_ = num_boxes[0];
|
||||
// The detection model with Detection_PostProcess op may output duplicate
|
||||
// boxes with different classes, in the following format:
|
||||
// num_boxes_tensor = [num_boxes]
|
||||
// detection_classes_tensor = [box_1_class_1, box_1_class_2, ...]
|
||||
// detection_scores_tensor = [box_1_score_1, box_1_score_2, ... ]
|
||||
// detection_boxes_tensor = [box_1, box1, ... ]
|
||||
// Each box repeats classes_per_detection_ times.
|
||||
// Note Detection_PostProcess op is only supported in CPU.
|
||||
RET_CHECK_EQ(max_detections % num_boxes_, 0);
|
||||
classes_per_detection_ = max_detections / num_boxes_;
|
||||
|
||||
auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView();
|
||||
auto detection_boxes = detection_boxes_view.buffer<float>();
|
||||
|
@ -493,8 +504,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
|
|||
|
||||
auto detection_classes_view = detection_classes_tensor->GetCpuReadView();
|
||||
auto detection_classes_ptr = detection_classes_view.buffer<float>();
|
||||
std::vector<int> detection_classes(num_boxes_);
|
||||
for (int i = 0; i < num_boxes_; ++i) {
|
||||
std::vector<int> detection_classes(num_boxes_ * classes_per_detection_);
|
||||
for (int i = 0; i < detection_classes.size(); ++i) {
|
||||
detection_classes[i] = static_cast<int>(detection_classes_ptr[i]);
|
||||
}
|
||||
MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores,
|
||||
|
@ -863,24 +874,25 @@ absl::Status TensorsToDetectionsCalculator::DecodeBoxes(
|
|||
absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
|
||||
const float* detection_boxes, const float* detection_scores,
|
||||
const int* detection_classes, std::vector<Detection>* output_detections) {
|
||||
for (int i = 0; i < num_boxes_; ++i) {
|
||||
for (int i = 0; i < num_boxes_ * classes_per_detection_;
|
||||
i += classes_per_detection_) {
|
||||
if (max_results_ > 0 && output_detections->size() == max_results_) {
|
||||
break;
|
||||
}
|
||||
if (options_.has_min_score_thresh() &&
|
||||
detection_scores[i] < options_.min_score_thresh()) {
|
||||
continue;
|
||||
}
|
||||
if (!IsClassIndexAllowed(detection_classes[i])) {
|
||||
continue;
|
||||
}
|
||||
const int box_offset = i * num_coords_;
|
||||
Detection detection = ConvertToDetection(
|
||||
/*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
|
||||
/*box_xmin=*/detection_boxes[box_offset + box_indices_[1]],
|
||||
/*box_ymax=*/detection_boxes[box_offset + box_indices_[2]],
|
||||
/*box_xmax=*/detection_boxes[box_offset + box_indices_[3]],
|
||||
detection_scores[i], detection_classes[i], options_.flip_vertically());
|
||||
absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
|
||||
absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
|
||||
options_.flip_vertically());
|
||||
// if all the scores and classes are filtered out, we skip the empty
|
||||
// detection.
|
||||
if (detection.score().empty()) {
|
||||
continue;
|
||||
}
|
||||
const auto& bbox = detection.location_data().relative_bounding_box();
|
||||
if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
|
||||
std::isnan(bbox.height())) {
|
||||
|
@ -910,11 +922,21 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
|
|||
}
|
||||
|
||||
Detection TensorsToDetectionsCalculator::ConvertToDetection(
|
||||
float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score,
|
||||
int class_id, bool flip_vertically) {
|
||||
float box_ymin, float box_xmin, float box_ymax, float box_xmax,
|
||||
absl::Span<const float> scores, absl::Span<const int> class_ids,
|
||||
bool flip_vertically) {
|
||||
Detection detection;
|
||||
detection.add_score(score);
|
||||
detection.add_label_id(class_id);
|
||||
for (int i = 0; i < scores.size(); ++i) {
|
||||
if (!IsClassIndexAllowed(class_ids[i])) {
|
||||
continue;
|
||||
}
|
||||
if (options_.has_min_score_thresh() &&
|
||||
scores[i] < options_.min_score_thresh()) {
|
||||
continue;
|
||||
}
|
||||
detection.add_score(scores[i]);
|
||||
detection.add_label_id(class_ids[i]);
|
||||
}
|
||||
|
||||
LocationData* location_data = detection.mutable_location_data();
|
||||
location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);
|
||||
|
|
|
@ -75,7 +75,7 @@ message TensorsToDetectionsCalculatorOptions {
|
|||
// representation has a bottom-left origin (e.g., in OpenGL).
|
||||
optional bool flip_vertically = 18 [default = false];
|
||||
|
||||
// Score threshold for perserving decoded detections.
|
||||
// Score threshold for preserving decoded detections.
|
||||
optional float min_score_thresh = 19;
|
||||
|
||||
// The maximum number of the detection results to return. If < 0, all
|
||||
|
|
|
@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
|
|||
kFlipVertically(cc).IsConnected())) {
|
||||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input width/height for using flipping when outputing "
|
||||
<< "Must provide input width/height for using flipping when outputting "
|
||||
"landmarks in absolute coordinates.";
|
||||
}
|
||||
return absl::OkStatus();
|
||||
|
|
|
@ -12,32 +12,35 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/gpu_buffer_format.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
#include "mediapipe/framework/formats/image_opencv.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
@ -62,37 +65,9 @@ namespace {
|
|||
constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
|
||||
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size) { // NOLINT
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
bool CanUseGpu() {
|
||||
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
// TODO: Configure GPU usage policy in individual calculators.
|
||||
constexpr bool kAllowGpuProcessing = true;
|
||||
return kAllowGpuProcessing;
|
||||
#else
|
||||
return false;
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
|
||||
constexpr char kMaskTag[] = "MASK";
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims) {
|
||||
if (dims.size() == 3) {
|
||||
return std::make_tuple(dims[0], dims[1], dims[2]);
|
||||
} else if (dims.size() == 4) {
|
||||
// BHWC format check B == 1
|
||||
RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
|
||||
return std::make_tuple(dims[1], dims[2], dims[3]);
|
||||
} else {
|
||||
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
|
|||
private:
|
||||
absl::Status LoadOptions(CalculatorContext* cc);
|
||||
absl::Status InitGpu(CalculatorContext* cc);
|
||||
absl::Status ProcessGpu(CalculatorContext* cc);
|
||||
absl::Status ProcessCpu(CalculatorContext* cc);
|
||||
absl::Status ProcessGpu(CalculatorContext* cc,
|
||||
const std::vector<Tensor>& input_tensors,
|
||||
std::tuple<int, int, int> hwc, int output_width,
|
||||
int output_height);
|
||||
void GlRender();
|
||||
|
||||
bool DoesGpuTextureStartAtBottom() {
|
||||
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
|
||||
}
|
||||
|
||||
absl::Status InitConverterIfNecessary() {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
template <class T>
|
||||
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
|
||||
if (!cpu_converter_) {
|
||||
MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
|
||||
}
|
||||
#else
|
||||
RET_CHECK_FAIL() << "OpenCV processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
::mediapipe::TensorsToSegmentationCalculatorOptions options_;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::TensorsToSegmentationCalculatorOptions options_;
|
||||
std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
|
@ -208,7 +192,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract(
|
|||
if (CanUseGpu()) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
|
||||
#endif // MEDIAPIPE_METAL_ENABLED
|
||||
|
@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
MP_ASSIGN_OR_RETURN(auto hwc,
|
||||
GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
int tensor_channels = std::get<2>(hwc);
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE:
|
||||
RET_CHECK_EQ(tensor_channels, 1);
|
||||
|
@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
}
|
||||
}
|
||||
|
||||
// Get dimensions.
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
if (use_gpu) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (!gpu_initialized_) {
|
||||
|
@ -286,16 +281,25 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
|
||||
MP_RETURN_IF_ERROR(ProcessGpu(cc));
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
MP_RETURN_IF_ERROR(
|
||||
gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
|
||||
output_height, hwc]() -> absl::Status {
|
||||
MP_RETURN_IF_ERROR(
|
||||
ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
#else
|
||||
RET_CHECK_FAIL() << "GPU processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
} else {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
MP_RETURN_IF_ERROR(ProcessCpu(cc));
|
||||
// Lazily initialize converter.
|
||||
MP_RETURN_IF_ERROR(InitConverterIfNecessary());
|
||||
MP_ASSIGN_OR_RETURN(
|
||||
std::unique_ptr<Image> output_mask,
|
||||
cpu_converter_->Convert(input_tensors, output_width, output_height));
|
||||
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
|
||||
cc->InputTimestamp());
|
||||
#else
|
||||
RET_CHECK_FAIL() << "OpenCV processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
|
|||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorsToSegmentationCalculator::ProcessCpu(
|
||||
CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
// Get input streams, and dimensions.
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
// Create initial working mask.
|
||||
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
|
||||
|
||||
// Wrap input tensor.
|
||||
auto raw_input_tensor = &input_tensors[0];
|
||||
auto raw_input_view = raw_input_tensor->GetCpuReadView();
|
||||
const float* raw_input_data = raw_input_view.buffer<float>();
|
||||
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
|
||||
CV_MAKETYPE(CV_32F, tensor_channels),
|
||||
const_cast<float*>(raw_input_data));
|
||||
|
||||
// Process mask tensor and apply activation function.
|
||||
if (tensor_channels == 2) {
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
|
||||
} else if (tensor_channels == 1) {
|
||||
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
|
||||
options_.activation()); // Requires 2 channels.
|
||||
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
|
||||
options_.activation()) // Pass-through optimization.
|
||||
tensor_mat.copyTo(small_mask_mat);
|
||||
else
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
|
||||
} else {
|
||||
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
|
||||
<< tensor_channels;
|
||||
}
|
||||
|
||||
// Send out image as CPU packet.
|
||||
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
|
||||
ImageFormat::VEC32F1, output_width, output_height);
|
||||
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
|
||||
auto output_mat = formats::MatView(output_mask.get());
|
||||
// Upsample small mask into output.
|
||||
cv::resize(small_mask_mat, *output_mat,
|
||||
cv::Size(output_width, output_height));
|
||||
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
template <class T>
|
||||
absl::Status TensorsToSegmentationCalculator::ApplyActivation(
|
||||
cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
|
||||
// Configure activation function.
|
||||
const int output_layer_index = options_.output_layer_index();
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
|
||||
float new_mask_value = 0;
|
||||
// TODO consider moving switch out of the loop,
|
||||
// and also avoid float/Vec2f casting.
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE: {
|
||||
new_mask_value = mask_value[0];
|
||||
break;
|
||||
}
|
||||
case Options::SIGMOID: {
|
||||
const float pixel0 = mask_value[0];
|
||||
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
|
||||
break;
|
||||
}
|
||||
case Options::SOFTMAX: {
|
||||
const float pixel0 = mask_value[0];
|
||||
const float pixel1 = mask_value[1];
|
||||
const float max_pixel = std::max(pixel0, pixel1);
|
||||
const float min_pixel = std::min(pixel0, pixel1);
|
||||
const float softmax_denom =
|
||||
/*exp(max_pixel - max_pixel)=*/1.0f +
|
||||
std::exp(min_pixel - max_pixel);
|
||||
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
|
||||
softmax_denom;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new_mask_value;
|
||||
};
|
||||
|
||||
// Process mask tensor.
|
||||
for (int i = 0; i < tensor_mat.rows; ++i) {
|
||||
for (int j = 0; j < tensor_mat.cols; ++j) {
|
||||
const T& input_pix = tensor_mat.at<T>(i, j);
|
||||
const float mask_value = activation_fn(input_pix);
|
||||
small_mask_mat->at<float>(i, j) = mask_value;
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
// Steps:
|
||||
// 1. receive tensor
|
||||
// 2. process segmentation tensor into small mask
|
||||
// 3. upsample small mask into output mask to be same size as input image
|
||||
absl::Status TensorsToSegmentationCalculator::ProcessGpu(
|
||||
CalculatorContext* cc) {
|
||||
CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
|
||||
std::tuple<int, int, int> hwc, int output_width, int output_height) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
// Get input streams, and dimensions.
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
// Create initial working mask texture.
|
||||
#if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
|
||||
|
@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() {
|
|||
absl::Status TensorsToSegmentationCalculator::LoadOptions(
|
||||
CalculatorContext* cc) {
|
||||
// Get calculator options specified in the graph.
|
||||
options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>();
|
||||
options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
@ -826,7 +713,7 @@ void main() {
|
|||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
// Shader defines.
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
const std::string output_layer_index =
|
||||
"\n#define OUTPUT_LAYER_INDEX int(" +
|
||||
std::to_string(options_.output_layer_index()) + ")";
|
||||
|
|
|
@ -17,10 +17,8 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/log/log.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
|
@ -30,7 +28,6 @@
|
|||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
|
@ -40,62 +37,17 @@ namespace {
|
|||
using ::testing::SizeIs;
|
||||
using ::testing::TestWithParam;
|
||||
using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
|
||||
|
||||
std::string ActivationTypeToString(Options::Activation activation) {
|
||||
switch (activation) {
|
||||
case Options::NONE:
|
||||
return "NONE";
|
||||
case Options::SIGMOID:
|
||||
return "SIGMOID";
|
||||
case Options::SOFTMAX:
|
||||
return "SOFTMAX";
|
||||
default:
|
||||
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
|
||||
return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
using TensorsToSegmentationCalculatorTest =
|
||||
TestWithParam<test_utils::FormattingTestCase>;
|
||||
|
||||
struct FormattingTestCase {
|
||||
std::string test_name;
|
||||
std::vector<float> inputs;
|
||||
std::vector<float> expected_outputs;
|
||||
Options::Activation activation;
|
||||
int rows;
|
||||
int cols;
|
||||
int channels;
|
||||
};
|
||||
|
||||
using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
|
||||
|
||||
// Currently only useable for tests with no output resize.
|
||||
TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
|
||||
const FormattingTestCase& test_case = GetParam();
|
||||
std::vector<float> inputs = test_case.inputs;
|
||||
std::vector<float> expected_outputs = test_case.expected_outputs;
|
||||
Options::Activation activation = test_case.activation;
|
||||
int rows = test_case.rows;
|
||||
int cols = test_case.cols;
|
||||
int channels = test_case.channels;
|
||||
const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
|
||||
rows_new, cols_new, channels, max_abs_diff] = GetParam();
|
||||
|
||||
std::string string_config = absl::Substitute(
|
||||
R"pb(
|
||||
input_stream: "tensors"
|
||||
input_stream: "size"
|
||||
node {
|
||||
calculator: "TensorsToSegmentationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
input_stream: "OUTPUT_SIZE:size"
|
||||
output_stream: "MASK:image_as_mask"
|
||||
options: {
|
||||
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
|
||||
activation: $0
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb",
|
||||
ActivationTypeToString(activation));
|
||||
auto graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(string_config);
|
||||
test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
|
||||
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
|
||||
|
@ -119,28 +71,34 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
|
|||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0))));
|
||||
}
|
||||
|
||||
// The output size is defined as pair(new_width, new_height).
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"size",
|
||||
mediapipe::Adopt(new std::pair<int, int>(rows, cols)).At(Timestamp(0))));
|
||||
"size", mediapipe::Adopt(new std::pair<int, int>(cols_new, rows_new))
|
||||
.At(Timestamp(0))));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
|
||||
ASSERT_THAT(output_packets, SizeIs(1));
|
||||
const Image& image_as_mask = output_packets[0].Get<Image>();
|
||||
EXPECT_FALSE(image_as_mask.UsesGpu());
|
||||
|
||||
std::shared_ptr<cv::Mat> result_mat = formats::MatView(&image_as_mask);
|
||||
EXPECT_EQ(result_mat->rows, rows);
|
||||
EXPECT_EQ(result_mat->cols, cols);
|
||||
EXPECT_EQ(result_mat->channels(), channels);
|
||||
EXPECT_EQ(result_mat->rows, rows_new);
|
||||
EXPECT_EQ(result_mat->cols, cols_new);
|
||||
EXPECT_EQ(result_mat->channels(), 1);
|
||||
|
||||
// Compare the real result with the expected result.
|
||||
cv::Mat expected_result = cv::Mat(
|
||||
rows, cols, CV_32FC1, const_cast<float*>(expected_outputs.data()));
|
||||
cv::Mat expected_result =
|
||||
cv::Mat(rows_new, cols_new, CV_32FC1,
|
||||
const_cast<float*>(expected_outputs.data()));
|
||||
cv::Mat diff;
|
||||
cv::absdiff(*result_mat, expected_result, diff);
|
||||
double max_val;
|
||||
cv::minMaxLoc(diff, nullptr, &max_val);
|
||||
// Expects the maximum absolute pixel-by-pixel difference is less than 1e-5.
|
||||
// This delta is for passthorugh accuracy only.
|
||||
EXPECT_LE(max_val, 1e-5);
|
||||
|
||||
// The max allowable diff between output and expected output varies between
|
||||
// tests.
|
||||
EXPECT_LE(max_val, max_abs_diff);
|
||||
|
||||
MP_ASSERT_OK(graph.CloseInputStream("tensors"));
|
||||
MP_ASSERT_OK(graph.CloseInputStream("size"));
|
||||
|
@ -149,18 +107,97 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
|
|||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
|
||||
testing::ValuesIn<FormattingTestCase>({
|
||||
{/*test_name=*/"NoActivationAndNoOutputResize",
|
||||
/*inputs=*/
|
||||
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
|
||||
14.0, 15.0, 16.0},
|
||||
/*expected_outputs=*/
|
||||
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
|
||||
14.0, 15.0, 16.0},
|
||||
/*activation=*/Options::NONE,
|
||||
/*rows=*/4,
|
||||
/*cols=*/4,
|
||||
/*channels=*/1},
|
||||
testing::ValuesIn<test_utils::FormattingTestCase>({
|
||||
{.test_name = "NoActivationAndNoOutputResize",
|
||||
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
|
||||
12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
.expected_outputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
|
||||
11.0, 12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
.activation = Options::NONE,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 4,
|
||||
.cols_new = 4,
|
||||
.channels = 1,
|
||||
.max_abs_diff = 1e-7},
|
||||
{.test_name = "OutputResizeOnly",
|
||||
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
|
||||
12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
.expected_outputs = {1, 1.5, 2.166667, 2.833333, 3.5, 4,
|
||||
3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8,
|
||||
7, 7.5, 8.166667, 8.833333, 9.5, 10,
|
||||
10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2,
|
||||
13, 13.5, 14.166667, 14.833333, 15.5, 16},
|
||||
.activation = Options::NONE,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 5,
|
||||
.cols_new = 6,
|
||||
.channels = 1,
|
||||
.max_abs_diff = 1e-6},
|
||||
{.test_name = "SigmoidActivationWithNoOutputResize",
|
||||
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
|
||||
12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
.expected_outputs = {0.731059, 0.880797, 0.952574, 0.982014, 0.993307,
|
||||
0.997527, 0.999089, 0.999665, 0.999877, 0.999955,
|
||||
0.999983, 0.999994, 0.999998, 0.999999, 1.0, 1.0},
|
||||
.activation = Options::SIGMOID,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 4,
|
||||
.cols_new = 4,
|
||||
.channels = 1,
|
||||
.max_abs_diff = 1e-6},
|
||||
{.test_name = "SigmoidActivationWithOutputResize",
|
||||
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
|
||||
12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
.expected_outputs = {0.731059, 0.805928, 0.89276, 0.940611, 0.967294,
|
||||
0.982014, 0.914633, 0.93857, 0.966279, 0.981363,
|
||||
0.989752, 0.994369, 0.996592, 0.997666, 0.998873,
|
||||
0.999404, 0.999683, 0.999829, 0.999913, 0.99994,
|
||||
0.999971, 0.999985, 0.999992, 0.999996, 0.999998,
|
||||
0.999998, 0.999999, 1.0, 1.0, 1.0},
|
||||
.activation = Options::SIGMOID,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 5,
|
||||
.cols_new = 6,
|
||||
.channels = 1,
|
||||
.max_abs_diff = 1e-6},
|
||||
{.test_name = "SoftmaxActivationWithNoOutputResize",
|
||||
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
|
||||
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
|
||||
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
|
||||
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
|
||||
.expected_outputs = {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574,
|
||||
0.000911051, 0.952574, 0.924142, 0.731059,
|
||||
0.731059, 0.24974, 0.937027, 0.689974, 0.990048,
|
||||
0.0060598, 0.28905},
|
||||
.activation = Options::SOFTMAX,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 4,
|
||||
.cols_new = 4,
|
||||
.channels = 2,
|
||||
.max_abs_diff = 1e-6},
|
||||
{.test_name = "SoftmaxActivationWithOutputResize",
|
||||
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
|
||||
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
|
||||
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
|
||||
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
|
||||
.expected_outputs = {0.731059, 0.425131, 0.246135, 0.753865, 0.445892,
|
||||
0.0109869, 0.886119, 0.461259, 0.185506, 0.781934,
|
||||
0.790618, 0.650195, 0.841816, 0.603901, 0.40518,
|
||||
0.561962, 0.765871, 0.930584, 0.718733, 0.763744,
|
||||
0.703402, 0.281989, 0.459635, 0.742634, 0.689974,
|
||||
0.840011, 0.82605, 0.170058, 0.147555, 0.28905},
|
||||
.activation = Options::SOFTMAX,
|
||||
.rows = 4,
|
||||
.cols = 4,
|
||||
.rows_new = 5,
|
||||
.cols_new = 6,
|
||||
.channels = 2,
|
||||
.max_abs_diff = 1e-6},
|
||||
}),
|
||||
[](const testing::TestParamInfo<
|
||||
TensorsToSegmentationCalculatorTest::ParamType>& info) {
|
||||
|
|
|
@ -0,0 +1,111 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tensors_to_segmentation_utils {
|
||||
|
||||
std::string ActivationTypeToString(
|
||||
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
|
||||
switch (activation) {
|
||||
case TensorsToSegmentationCalculatorOptions::NONE:
|
||||
return "NONE";
|
||||
case TensorsToSegmentationCalculatorOptions::SIGMOID:
|
||||
return "SIGMOID";
|
||||
case TensorsToSegmentationCalculatorOptions::SOFTMAX:
|
||||
return "SOFTMAX";
|
||||
}
|
||||
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
std::vector<unsigned char> ArrayFloatToUnsignedChar(
|
||||
const std::vector<float>& array) {
|
||||
std::vector<unsigned char> result;
|
||||
result.reserve(array.size());
|
||||
for (int i = 0; i < array.size(); ++i) {
|
||||
result.push_back(static_cast<unsigned char>(array[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
|
||||
std::vector<float> result;
|
||||
result.reserve(values.size() * 4);
|
||||
for (const float& value : values) {
|
||||
result.push_back(value);
|
||||
result.push_back(0);
|
||||
result.push_back(0);
|
||||
result.push_back(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// For GPU tests, the input tensor needs to be moved to GPU, using
|
||||
// TensorViewRequestor. After calculation, the output needs to be moved back
|
||||
// to CPU, using ToImageCalculator. The output is an ImageFrame.
|
||||
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
|
||||
bool test_gpu,
|
||||
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
|
||||
std::string pre_process = R"pb(
|
||||
node {
|
||||
calculator: "mediapipe.aimatter.TensorViewRequestor"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "TENSORS:tensors_gpu"
|
||||
options {
|
||||
[mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
|
||||
}
|
||||
}
|
||||
)pb";
|
||||
std::string post_process = R"pb(
|
||||
node {
|
||||
calculator: "FromImageCalculator"
|
||||
input_stream: "IMAGE:image_as_mask_gpu"
|
||||
output_stream: "IMAGE_CPU:image_as_mask"
|
||||
}
|
||||
)pb";
|
||||
return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
|
||||
absl::Substitute(
|
||||
R"pb(
|
||||
input_stream: "tensors"
|
||||
input_stream: "size" $0
|
||||
node {
|
||||
calculator: "TensorsToSegmentationCalculator"
|
||||
input_stream: "TENSORS:tensors$1"
|
||||
input_stream: "OUTPUT_SIZE:size"
|
||||
output_stream: "MASK:image_as_mask$2"
|
||||
options: {
|
||||
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
|
||||
activation: $3
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
} $4
|
||||
)pb",
|
||||
test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
|
||||
test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
|
||||
test_gpu ? post_process : ""));
|
||||
}
|
||||
} // namespace tensors_to_segmentation_utils
|
||||
} // namespace mediapipe
|
|
@ -0,0 +1,57 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tensors_to_segmentation_utils {
|
||||
std::string ActivationTypeToString(
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
|
||||
activation);
|
||||
|
||||
std::vector<unsigned char> ArrayFloatToUnsignedChar(
|
||||
const std::vector<float>& array);
|
||||
|
||||
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
|
||||
|
||||
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
|
||||
bool test_gpu,
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
|
||||
activation);
|
||||
|
||||
struct FormattingTestCase {
|
||||
std::string test_name;
|
||||
std::vector<float> inputs;
|
||||
std::vector<float> expected_outputs;
|
||||
mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
|
||||
int rows = 1;
|
||||
int cols = 1;
|
||||
int rows_new = 1;
|
||||
int cols_new = 1;
|
||||
int channels = 1;
|
||||
double max_abs_diff = 1e-7;
|
||||
};
|
||||
} // namespace tensors_to_segmentation_utils
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
|
@ -0,0 +1,50 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe::tensors_to_segmentation_utils {
|
||||
namespace {
|
||||
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
ActivationTypeToStringWorksCorrectly) {
|
||||
EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
|
||||
EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
|
||||
EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
ArrayFloatToUnsignedCharWorksCorrectly) {
|
||||
std::vector<float> input = {1.0, 2.0, 3.0};
|
||||
std::vector<unsigned char> expected = {1, 2, 3};
|
||||
EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
MakeRedAlphaMatrixWorksCorrectly) {
|
||||
std::vector<float> input = {1.0, 2.0, 3.0};
|
||||
std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
|
||||
0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
|
||||
EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe::tensors_to_segmentation_utils
|
|
@ -0,0 +1,43 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class TensorsToSegmentationConverter {
|
||||
public:
|
||||
virtual ~TensorsToSegmentationConverter() = default;
|
||||
|
||||
// Converts tensors to image mask.
|
||||
// Returns a unique pointer containing the converted image.
|
||||
// @input_tensors contains the tensors needed to be processed.
|
||||
// @output_width/height describes output dimensions to reshape the output mask
|
||||
// into.
|
||||
virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) = 0;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
|
@ -0,0 +1,157 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_opencv.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
class OpenCvProcessor : public TensorsToSegmentationConverter {
|
||||
public:
|
||||
absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
|
||||
options_ = options;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::StatusOr<std::unique_ptr<Image>> Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) override;
|
||||
|
||||
private:
|
||||
template <class T>
|
||||
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
|
||||
|
||||
TensorsToSegmentationCalculatorOptions options_;
|
||||
};
|
||||
|
||||
absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) {
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
// Create initial working mask.
|
||||
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
|
||||
|
||||
// Wrap input tensor.
|
||||
auto raw_input_tensor = &input_tensors[0];
|
||||
auto raw_input_view = raw_input_tensor->GetCpuReadView();
|
||||
const float* raw_input_data = raw_input_view.buffer<float>();
|
||||
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
|
||||
CV_MAKETYPE(CV_32F, tensor_channels),
|
||||
const_cast<float*>(raw_input_data));
|
||||
|
||||
// Process mask tensor and apply activation function.
|
||||
if (tensor_channels == 2) {
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
|
||||
} else if (tensor_channels == 1) {
|
||||
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
|
||||
options_.activation()); // Requires 2 channels.
|
||||
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
|
||||
options_.activation()) // Pass-through optimization.
|
||||
tensor_mat.copyTo(small_mask_mat);
|
||||
else
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
|
||||
} else {
|
||||
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
|
||||
<< tensor_channels;
|
||||
}
|
||||
|
||||
// Send out image as CPU packet.
|
||||
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
|
||||
ImageFormat::VEC32F1, output_width, output_height);
|
||||
auto output_mask = std::make_unique<Image>(mask_frame);
|
||||
auto output_mat = formats::MatView(output_mask.get());
|
||||
// Upsample small mask into output.
|
||||
cv::resize(small_mask_mat, *output_mat,
|
||||
cv::Size(output_width, output_height));
|
||||
return output_mask;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
|
||||
cv::Mat* small_mask_mat) {
|
||||
// Configure activation function.
|
||||
const int output_layer_index = options_.output_layer_index();
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
|
||||
float new_mask_value = 0;
|
||||
// TODO consider moving switch out of the loop,
|
||||
// and also avoid float/Vec2f casting.
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE: {
|
||||
new_mask_value = mask_value[0];
|
||||
break;
|
||||
}
|
||||
case Options::SIGMOID: {
|
||||
const float pixel0 = mask_value[0];
|
||||
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
|
||||
break;
|
||||
}
|
||||
case Options::SOFTMAX: {
|
||||
const float pixel0 = mask_value[0];
|
||||
const float pixel1 = mask_value[1];
|
||||
const float max_pixel = std::max(pixel0, pixel1);
|
||||
const float min_pixel = std::min(pixel0, pixel1);
|
||||
const float softmax_denom =
|
||||
/*exp(max_pixel - max_pixel)=*/1.0f +
|
||||
std::exp(min_pixel - max_pixel);
|
||||
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
|
||||
softmax_denom;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new_mask_value;
|
||||
};
|
||||
|
||||
// Process mask tensor.
|
||||
for (int i = 0; i < tensor_mat.rows; ++i) {
|
||||
for (int j = 0; j < tensor_mat.cols; ++j) {
|
||||
const T& input_pix = tensor_mat.at<T>(i, j);
|
||||
const float mask_value = activation_fn(input_pix);
|
||||
small_mask_mat->at<float>(i, j) = mask_value;
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
|
||||
CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
|
||||
auto converter = std::make_unique<OpenCvProcessor>();
|
||||
MP_RETURN_IF_ERROR(converter->Init(options));
|
||||
return converter;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
|
@ -0,0 +1,31 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
// Creates OpenCV tensors-to-segmentation converter.
|
||||
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
|
||||
CreateOpenCvConverter(
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions& options);
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
|
@ -0,0 +1,52 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
int NumGroups(int size, int group_size) {
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
bool CanUseGpu() {
|
||||
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
// TODO: Configure GPU usage policy in individual calculators.
|
||||
constexpr bool kAllowGpuProcessing = true;
|
||||
return kAllowGpuProcessing;
|
||||
#else
|
||||
return false;
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims) {
|
||||
if (dims.size() == 3) {
|
||||
return std::make_tuple(dims[0], dims[1], dims[2]);
|
||||
} else if (dims.size() == 4) {
|
||||
// BHWC format check B == 1
|
||||
RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
|
||||
return std::make_tuple(dims[1], dims[2], dims[3]);
|
||||
} else {
|
||||
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
|
||||
}
|
||||
}
|
||||
} // namespace mediapipe
|
34
mediapipe/calculators/tensor/tensors_to_segmentation_utils.h
Normal file
34
mediapipe/calculators/tensor/tensors_to_segmentation_utils.h
Normal file
|
@ -0,0 +1,34 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size); // NOLINT
|
||||
|
||||
bool CanUseGpu();
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims);
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
|
@ -0,0 +1,63 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using ::testing::HasSubstr;
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
|
||||
EXPECT_EQ(NumGroups(13, 4), 4);
|
||||
EXPECT_EQ(NumGroups(4, 13), 1);
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
|
||||
std::vector<int> dims_3 = {2, 3, 4};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
|
||||
MP_ASSERT_OK(result_1);
|
||||
EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
|
||||
std::vector<int> dims_4 = {1, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
|
||||
MP_ASSERT_OK(result_2);
|
||||
EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
|
||||
std::vector<int> dims_4 = {2, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
|
||||
EXPECT_FALSE(result.ok());
|
||||
EXPECT_THAT(result.status().message(),
|
||||
HasSubstr("Expected batch to be 1 for BHWC heatmap"));
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
|
||||
std::vector<int> dims_5 = {1, 2, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
|
||||
EXPECT_FALSE(result.ok());
|
||||
EXPECT_THAT(result.status().message(),
|
||||
HasSubstr("Invalid shape for segmentation tensor"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
|
@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
|
|||
// and label and label_id are optional but at least one of them should be set.
|
||||
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
|
||||
// prefixed versions of each stream, which allows for multiple image streams to
|
||||
// be included. However, the default names are suppored by more tools.
|
||||
// be included. However, the default names are supported by more tools.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
|
|
|
@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
|
|||
// -- 1-D or 2-D Tensor
|
||||
// Output:
|
||||
// -- Matrix with the same values as the Tensor
|
||||
// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape.
|
||||
// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape.
|
||||
// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
|
||||
// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
|
||||
//
|
||||
// Example Config
|
||||
// node: {
|
||||
|
|
|
@ -15,9 +15,9 @@
|
|||
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
|
||||
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/base/integral_types.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
|
|
@ -111,8 +111,8 @@ class InferenceState {
|
|||
// input_side_packet.
|
||||
//
|
||||
// The input and output streams are TensorFlow tensors labeled by tags. The tags
|
||||
// for the streams are matched to feeds and fetchs in a TensorFlow session using
|
||||
// a named_signature.generic_signature in the ModelManifest. The
|
||||
// for the streams are matched to feeds and fetches in a TensorFlow session
|
||||
// using a named_signature.generic_signature in the ModelManifest. The
|
||||
// generic_signature is used as key-value pairs between the MediaPipe tag and
|
||||
// the TensorFlow tensor. The signature_name in the options proto determines
|
||||
// which named_signature is used. The keys in the generic_signature must be
|
||||
|
@ -128,7 +128,7 @@ class InferenceState {
|
|||
// addition. Once batch_size inputs have been provided, the batch will be run
|
||||
// and the output tensors sent out on the output streams with timestamps
|
||||
// corresponding to the input stream packets. Setting the batch_size to 1
|
||||
// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
|
||||
// completely disables batching, but is independent of add_batch_dim_to_tensors.
|
||||
//
|
||||
// The TensorFlowInferenceCalculator also support feeding states recurrently for
|
||||
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the
|
||||
|
|
|
@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
|
|||
// If the 0th dimension is the batch dimension, then the tensors are
|
||||
// concatenated on that dimension. If the 0th is a data dimension, then a 0th
|
||||
// dimension is added before concatenating. If added, the extra dimension is
|
||||
// removed before outputing the tensor. Examples of each case: If you want
|
||||
// removed before outputting the tensor. Examples of each case: If you want
|
||||
// to batch spectra of audio over time for an LSTM, a time-frequency
|
||||
// representation has a 0th dimension as the batch dimension. If you want to
|
||||
// batch frames of video that are [width, height, channels], the batch
|
||||
|
|
|
@ -1,2 +1,2 @@
|
|||
The model files add.bin, add_quantized.bin
|
||||
(and corresponding metatada json files) come from tensorflow/lite/testdata/
|
||||
(and corresponding metadata json files) come from tensorflow/lite/testdata/
|
||||
|
|
|
@ -95,7 +95,7 @@ struct GPUData {
|
|||
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
|
||||
// or MTLBuffer.
|
||||
//
|
||||
// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
|
||||
// This calculator is designed to be used with the TfLiteInferenceCalculator,
|
||||
// as a pre-processing step for calculator inputs.
|
||||
//
|
||||
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],
|
||||
|
|
|
@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
|
|||
// Custom settings to override the internal scaling factors `div` and `sub`.
|
||||
// Both values must be set to non-negative values. Will only take effect on
|
||||
// CPU AND when |use_custom_normalization| is set to true. When these custom
|
||||
// values take effect, the |zero_center| setting above will be overriden, and
|
||||
// values take effect, the |zero_center| setting above will be overridden, and
|
||||
// the normalized_value will be calculated as:
|
||||
// normalized_value = input / custom_div - custom_sub.
|
||||
optional bool use_custom_normalization = 6 [default = false];
|
||||
|
|
|
@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
|
|||
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
|
||||
}
|
||||
|
||||
// Score threshold for perserving the class.
|
||||
// Score threshold for preserving the class.
|
||||
optional float min_score_threshold = 1;
|
||||
// Number of highest scoring labels to output. If top_k is not positive then
|
||||
// all labels are used.
|
||||
|
|
|
@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
|
|||
// tensors can have 2 or 3 tensors. First tensor is the predicted
|
||||
// raw boxes/keypoints. The size of the values must be (num_boxes
|
||||
// * num_predicted_values). Second tensor is the score tensor. The
|
||||
// size of the valuse must be (num_boxes * num_classes). It's
|
||||
// size of the values must be (num_boxes * num_classes). It's
|
||||
// optional to pass in a third tensor for anchors (e.g. for SSD
|
||||
// models) depend on the outputs of the detection model. The size
|
||||
// of anchor tensor must be (num_boxes * 4).
|
||||
|
|
|
@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
|
|||
// representation has a bottom-left origin (e.g., in OpenGL).
|
||||
optional bool flip_vertically = 18 [default = false];
|
||||
|
||||
// Score threshold for perserving decoded detections.
|
||||
// Score threshold for preserving decoded detections.
|
||||
optional float min_score_thresh = 19;
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
|
|||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input width/height for using flip_vertically option "
|
||||
"when outputing landmarks in absolute coordinates.";
|
||||
"when outputting landmarks in absolute coordinates.";
|
||||
}
|
||||
|
||||
flip_horizontally_ =
|
||||
|
|
Binary file not shown.
|
@ -1,6 +1,7 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
29
mediapipe/examples/android/solutions/gradlew
vendored
29
mediapipe/examples/android/solutions/gradlew
vendored
|
@ -83,10 +83,8 @@ done
|
|||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
@ -133,10 +131,13 @@ location of your Java installation."
|
|||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
|
@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
|||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC3045
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
|
@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
|||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC3045
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
|
@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
|
|||
done
|
||||
fi
|
||||
|
||||
# Collect all arguments for the java command;
|
||||
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
|
||||
# shell script including quotes and variable substitutions, so put them in
|
||||
# double quotes to make sure that they get re-expanded; and
|
||||
# * put everything else in single quotes, so that it's not re-expanded.
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
|
|
|
@ -55,7 +55,7 @@ absl::Status RunMPPGraph() {
|
|||
for (const std::string& kv_pair : kv_pairs) {
|
||||
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
|
||||
RET_CHECK(name_and_value.size() == 2);
|
||||
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
|
||||
RET_CHECK(!input_side_packets.contains(name_and_value[0]));
|
||||
std::string input_side_packet_contents;
|
||||
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
|
||||
name_and_value[1], &input_side_packet_contents));
|
||||
|
|
|
@ -56,7 +56,7 @@ absl::Status RunMPPGraph() {
|
|||
for (const std::string& kv_pair : kv_pairs) {
|
||||
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
|
||||
RET_CHECK(name_and_value.size() == 2);
|
||||
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
|
||||
RET_CHECK(!input_side_packets.contains(name_and_value[0]));
|
||||
std::string input_side_packet_contents;
|
||||
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
|
||||
name_and_value[1], &input_side_packet_contents));
|
||||
|
|
|
@ -330,6 +330,14 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
|
|||
|
||||
class NodeBase {
|
||||
public:
|
||||
NodeBase() = default;
|
||||
~NodeBase() = default;
|
||||
NodeBase(NodeBase&&) = default;
|
||||
NodeBase& operator=(NodeBase&&) = default;
|
||||
// Explicitly delete copies to improve error messages.
|
||||
NodeBase(const NodeBase&) = delete;
|
||||
NodeBase& operator=(const NodeBase&) = delete;
|
||||
|
||||
// TODO: right now access to an indexed port is made directly by
|
||||
// specifying both a tag and an index. It would be better to represent this
|
||||
// as a two-step lookup, first getting a multi-port, and then accessing one
|
||||
|
@ -585,6 +593,14 @@ class PacketGenerator {
|
|||
|
||||
class Graph {
|
||||
public:
|
||||
Graph() = default;
|
||||
~Graph() = default;
|
||||
Graph(Graph&&) = default;
|
||||
Graph& operator=(Graph&&) = default;
|
||||
// Explicitly delete copies to improve error messages.
|
||||
Graph(const Graph&) = delete;
|
||||
Graph& operator=(const Graph&) = delete;
|
||||
|
||||
void SetType(std::string type) { type_ = std::move(type); }
|
||||
|
||||
// Creates a node of a specific type. Should be used for calculators whose
|
||||
|
|
|
@ -124,6 +124,15 @@ cc_library(
|
|||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "ahwb_view",
|
||||
hdrs = ["ahwb_view.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/gpu:gpu_buffer_storage",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "affine_transform",
|
||||
srcs = ["affine_transform.cc"],
|
||||
|
|
54
mediapipe/framework/formats/ahwb_view.h
Normal file
54
mediapipe/framework/formats/ahwb_view.h
Normal file
|
@ -0,0 +1,54 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
|
||||
#include <android/hardware_buffer.h>
|
||||
|
||||
#include "mediapipe/gpu/gpu_buffer_storage.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
|
||||
// Intended use cases:
|
||||
// - Extracting an AHWB for processing in another library after it's produced by
|
||||
// MediaPipe.
|
||||
// - Sending AHWBs to compute devices that are able to map the memory for their
|
||||
// own usage.
|
||||
// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
|
||||
// other CPU/GPU uses of AHWBs.
|
||||
class AhwbView {
|
||||
public:
|
||||
explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
|
||||
// Non-copyable
|
||||
AhwbView(const AhwbView&) = delete;
|
||||
AhwbView& operator=(const AhwbView&) = delete;
|
||||
// Non-movable
|
||||
AhwbView(AhwbView&&) = delete;
|
||||
|
||||
// Only supports synchronous usage. All users of GetHandle must finish
|
||||
// accessing the buffer before this view object is destroyed to avoid race
|
||||
// conditions.
|
||||
// TODO: Support asynchronous usage.
|
||||
const AHardwareBuffer* GetHandle() const { return handle_; }
|
||||
|
||||
private:
|
||||
const AHardwareBuffer* handle_;
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
// Makes this class available as a GpuBuffer view.
|
||||
template <>
|
||||
class ViewProvider<AhwbView> {
|
||||
public:
|
||||
virtual ~ViewProvider() = default;
|
||||
virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
|
||||
virtual AhwbView GetWriteView(types<AhwbView>) = 0;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB
|
||||
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
|
@ -50,7 +50,7 @@
|
|||
// but may or may not still be able to run other OpenGL code.
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
|
||||
(defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
|
||||
MEDIAPIPE_USING_SWIFTSHADER)
|
||||
MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
|
||||
#define MEDIAPIPE_DISABLE_GL_COMPUTE
|
||||
#endif
|
||||
|
||||
|
@ -104,4 +104,9 @@
|
|||
#endif
|
||||
#endif // MEDIAPIPE_HAS_RTTI
|
||||
|
||||
// AHardware buffers are only available since Android API 26.
|
||||
#if (__ANDROID_API__ >= 26)
|
||||
#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
|
||||
#endif
|
||||
|
||||
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_
|
||||
|
|
|
@ -616,6 +616,7 @@ cc_test(
|
|||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/functional:bind_front",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
@ -856,6 +857,7 @@ cc_library(
|
|||
mediapipe_cc_test(
|
||||
name = "switch_demux_calculator_test",
|
||||
srcs = ["switch_demux_calculator_test.cc"],
|
||||
requires_full_emulation = False,
|
||||
deps = [
|
||||
":container_util",
|
||||
":switch_demux_calculator",
|
||||
|
@ -891,6 +893,7 @@ cc_library(
|
|||
mediapipe_cc_test(
|
||||
name = "switch_mux_calculator_test",
|
||||
srcs = ["switch_mux_calculator_test.cc"],
|
||||
requires_full_emulation = False,
|
||||
deps = [
|
||||
":container_util",
|
||||
":switch_mux_calculator",
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/functional/bind_front.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
|
|
|
@ -134,7 +134,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
|
|||
RET_CHECK(name);
|
||||
absl::Status tag_status = absl::OkStatus();
|
||||
absl::Status name_status = absl::UnknownError("");
|
||||
int name_index = 0;
|
||||
int name_index = -1;
|
||||
std::vector<std::string> v = absl::StrSplit(tag_and_name, ':');
|
||||
if (v.size() == 1) {
|
||||
name_status = ValidateName(v[0]);
|
||||
|
@ -143,7 +143,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
|
|||
tag_status = ValidateTag(v[0]);
|
||||
name_status = ValidateName(v[1]);
|
||||
name_index = 1;
|
||||
}
|
||||
} // else omitted, name_index == -1, triggering error.
|
||||
if (name_index == -1 || tag_status != absl::OkStatus() ||
|
||||
name_status != absl::OkStatus()) {
|
||||
tag->clear();
|
||||
|
|
|
@ -511,11 +511,19 @@ cc_library(
|
|||
],
|
||||
}),
|
||||
deps = [
|
||||
":gl_base_hdr",
|
||||
":gl_context",
|
||||
":gl_texture_buffer",
|
||||
":gl_texture_view",
|
||||
":gpu_buffer_format",
|
||||
":gpu_buffer_storage",
|
||||
":image_frame_view",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/formats:ahwb_view",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//third_party/GL:EGL_headers",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
],
|
||||
)
|
||||
|
@ -526,12 +534,14 @@ mediapipe_proto_library(
|
|||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
cc_library(
|
||||
name = "pixel_buffer_pool_util",
|
||||
srcs = ["pixel_buffer_pool_util.mm"],
|
||||
srcs = ["pixel_buffer_pool_util.cc"],
|
||||
hdrs = ["pixel_buffer_pool_util.h"],
|
||||
copts = [
|
||||
"-x objective-c++",
|
||||
"-Wno-shorten-64-to-32",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
|
@ -542,13 +552,14 @@ objc_library(
|
|||
],
|
||||
)
|
||||
|
||||
objc_library(
|
||||
cc_library(
|
||||
name = "metal_shared_resources",
|
||||
srcs = ["metal_shared_resources.mm"],
|
||||
srcs = ["metal_shared_resources.cc"],
|
||||
hdrs = ["metal_shared_resources.h"],
|
||||
copts = [
|
||||
"-x objective-c++",
|
||||
"-Wno-shorten-64-to-32",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
features = ["-layering_check"],
|
||||
visibility = ["//visibility:public"],
|
||||
|
@ -557,15 +568,17 @@ objc_library(
|
|||
"@google_toolbox_for_mac//:GTM_Defines",
|
||||
] + [
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
objc_library(
|
||||
cc_library(
|
||||
name = "MPPMetalUtil",
|
||||
srcs = ["MPPMetalUtil.mm"],
|
||||
srcs = ["MPPMetalUtil.cc"],
|
||||
hdrs = ["MPPMetalUtil.h"],
|
||||
copts = [
|
||||
"-x objective-c++",
|
||||
"-Wno-shorten-64-to-32",
|
||||
"-fobjc-arc", # enable reference-counting
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
|
@ -575,6 +588,7 @@ objc_library(
|
|||
"@com_google_absl//absl/time",
|
||||
"@google_toolbox_for_mac//:GTM_Defines",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
|
@ -655,6 +669,8 @@ cc_library(
|
|||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/base:core_headers",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/status",
|
||||
] + select({
|
||||
"//conditions:default": [],
|
||||
"//mediapipe:apple": [
|
||||
|
@ -857,12 +873,14 @@ cc_library(
|
|||
}),
|
||||
)
|
||||
|
||||
objc_library(
|
||||
cc_library(
|
||||
name = "MPPMetalHelper",
|
||||
srcs = ["MPPMetalHelper.mm"],
|
||||
srcs = ["MPPMetalHelper.cc"],
|
||||
hdrs = ["MPPMetalHelper.h"],
|
||||
copts = [
|
||||
"-Wno-shorten-64-to-32",
|
||||
"-x objective-c++",
|
||||
"-fobjc-arc",
|
||||
],
|
||||
features = ["-layering_check"],
|
||||
visibility = ["//visibility:public"],
|
||||
|
@ -1215,9 +1233,13 @@ mediapipe_cc_test(
|
|||
],
|
||||
requires_full_emulation = True,
|
||||
deps = [
|
||||
":gl_texture_buffer",
|
||||
":gl_texture_util",
|
||||
":gpu_buffer_format",
|
||||
":gpu_buffer_storage_ahwb",
|
||||
":gpu_test_base",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/tool:test_util",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -14,15 +14,14 @@
|
|||
|
||||
#import "mediapipe/gpu/MPPMetalHelper.h"
|
||||
|
||||
#import "GTMDefines.h"
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#import "mediapipe/gpu/gpu_buffer.h"
|
||||
#import "mediapipe/gpu/gpu_service.h"
|
||||
#import "mediapipe/gpu/graph_support.h"
|
||||
#import "mediapipe/gpu/metal_shared_resources.h"
|
||||
#import "GTMDefines.h"
|
||||
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
@interface MPPMetalHelper () {
|
||||
mediapipe::GpuResources* _gpuResources;
|
||||
|
@ -31,7 +30,8 @@
|
|||
|
||||
namespace mediapipe {
|
||||
|
||||
// Using a C++ class so it can be declared as a friend of LegacyCalculatorSupport.
|
||||
// Using a C++ class so it can be declared as a friend of
|
||||
// LegacyCalculatorSupport.
|
||||
class MetalHelperLegacySupport {
|
||||
public:
|
||||
static CalculatorContract* GetCalculatorContract() {
|
||||
|
@ -61,7 +61,8 @@ class MetalHelperLegacySupport {
|
|||
|
||||
- (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc {
|
||||
if (!cc) return nil;
|
||||
return [self initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
|
||||
return [self
|
||||
initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
|
||||
}
|
||||
|
||||
+ (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc {
|
||||
|
@ -77,7 +78,8 @@ class MetalHelperLegacySupport {
|
|||
}
|
||||
|
||||
// Legacy support.
|
||||
- (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets {
|
||||
- (instancetype)initWithSidePackets:
|
||||
(const mediapipe::PacketSet&)inputSidePackets {
|
||||
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext();
|
||||
if (cc) {
|
||||
ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets());
|
||||
|
@ -85,16 +87,19 @@ class MetalHelperLegacySupport {
|
|||
}
|
||||
|
||||
// TODO: remove when we can.
|
||||
ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses "
|
||||
"CalculatorBase, call initWithCalculatorContext instead.";
|
||||
ABSL_LOG(WARNING)
|
||||
<< "CalculatorContext not available. If this calculator uses "
|
||||
"CalculatorBase, call initWithCalculatorContext instead.";
|
||||
mediapipe::GpuSharedData* gpu_shared =
|
||||
inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get<mediapipe::GpuSharedData*>();
|
||||
inputSidePackets.Tag(mediapipe::kGpuSharedTagName)
|
||||
.Get<mediapipe::GpuSharedData*>();
|
||||
|
||||
return [self initWithGpuResources:gpu_shared->gpu_resources.get()];
|
||||
}
|
||||
|
||||
// Legacy support.
|
||||
+ (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets {
|
||||
+ (absl::Status)setupInputSidePackets:
|
||||
(mediapipe::PacketTypeSet*)inputSidePackets {
|
||||
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract();
|
||||
if (cc) {
|
||||
ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets());
|
||||
|
@ -102,12 +107,12 @@ class MetalHelperLegacySupport {
|
|||
}
|
||||
|
||||
// TODO: remove when we can.
|
||||
ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this "
|
||||
"from a GetContract method, call updateContract instead.";
|
||||
ABSL_LOG(WARNING)
|
||||
<< "CalculatorContract not available. If you're calling this "
|
||||
"from a GetContract method, call updateContract instead.";
|
||||
auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0);
|
||||
RET_CHECK(id.IsValid())
|
||||
<< "A " << mediapipe::kGpuSharedTagName
|
||||
<< " input side packet is required here.";
|
||||
RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName
|
||||
<< " input side packet is required here.";
|
||||
inputSidePackets->Get(id).Set<mediapipe::GpuSharedData*>();
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
@ -125,10 +130,12 @@ class MetalHelperLegacySupport {
|
|||
}
|
||||
|
||||
- (id<MTLCommandBuffer>)commandBuffer {
|
||||
return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
|
||||
return
|
||||
[_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
|
||||
}
|
||||
|
||||
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
|
||||
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
|
||||
(const mediapipe::GpuBuffer&)gpuBuffer
|
||||
plane:(size_t)plane {
|
||||
CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer);
|
||||
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
|
||||
|
@ -178,41 +185,48 @@ class MetalHelperLegacySupport {
|
|||
CVMetalTextureRef texture;
|
||||
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
|
||||
NULL, _gpuResources->metal_shared().resources().mtlTextureCache,
|
||||
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane,
|
||||
&texture);
|
||||
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width,
|
||||
height, plane, &texture);
|
||||
ABSL_CHECK_EQ(err, kCVReturnSuccess);
|
||||
return texture;
|
||||
}
|
||||
|
||||
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
|
||||
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
|
||||
(const mediapipe::GpuBuffer&)gpuBuffer {
|
||||
return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0];
|
||||
}
|
||||
|
||||
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
|
||||
- (id<MTLTexture>)metalTextureWithGpuBuffer:
|
||||
(const mediapipe::GpuBuffer&)gpuBuffer {
|
||||
return [self metalTextureWithGpuBuffer:gpuBuffer plane:0];
|
||||
}
|
||||
|
||||
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
|
||||
plane:(size_t)plane {
|
||||
- (id<MTLTexture>)metalTextureWithGpuBuffer:
|
||||
(const mediapipe::GpuBuffer&)gpuBuffer
|
||||
plane:(size_t)plane {
|
||||
CFHolder<CVMetalTextureRef> cvTexture;
|
||||
cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]);
|
||||
return CVMetalTextureGetTexture(*cvTexture);
|
||||
}
|
||||
|
||||
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height {
|
||||
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
|
||||
height:(int)height {
|
||||
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height);
|
||||
}
|
||||
|
||||
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
|
||||
height:(int)height
|
||||
format:(mediapipe::GpuBufferFormat)format {
|
||||
height:(int)height
|
||||
format:(mediapipe::GpuBufferFormat)
|
||||
format {
|
||||
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format);
|
||||
}
|
||||
|
||||
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error {
|
||||
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name
|
||||
error:(NSError* _Nullable*)error {
|
||||
return [_gpuResources->metal_shared().resources().mtlDevice
|
||||
newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name
|
||||
ofType:@"metallib"]
|
||||
newLibraryWithFile:[[NSBundle bundleForClass:[self class]]
|
||||
pathForResource:name
|
||||
ofType:@"metallib"]
|
||||
error:error];
|
||||
}
|
||||
|
|
@ -69,10 +69,10 @@
|
|||
while (!bufferCompleted) {
|
||||
auto duration = absl::Now() - start_time;
|
||||
// If the spin-lock takes more than 5 ms then go to blocking wait:
|
||||
// - it frees the CPU core for another threads: increase the performance/decrease power
|
||||
// consumption.
|
||||
// - if a driver thread that notifies that the GPU buffer is completed has lower priority then
|
||||
// the CPU core is allocated for the thread.
|
||||
// - it frees the CPU core for another threads: increase the
|
||||
// performance/decrease power consumption.
|
||||
// - if a driver thread that notifies that the GPU buffer is completed has
|
||||
// lower priority then the CPU core is allocated for the thread.
|
||||
if (duration >= absl::Milliseconds(5)) {
|
||||
[commandBuffer waitUntilCompleted];
|
||||
break;
|
|
@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
|
|||
|
||||
// static
|
||||
absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
|
||||
bool requesst_gpu_as_optional) {
|
||||
if (requesst_gpu_as_optional) {
|
||||
bool request_gpu_as_optional) {
|
||||
if (request_gpu_as_optional) {
|
||||
cc->UseService(kGpuService).Optional();
|
||||
} else {
|
||||
cc->UseService(kGpuService);
|
||||
|
|
|
@ -68,7 +68,7 @@ class GlCalculatorHelper {
|
|||
// This method can be called from GetContract to set up the needed GPU
|
||||
// resources.
|
||||
static absl::Status UpdateContract(CalculatorContract* cc,
|
||||
bool requesst_gpu_as_optional = false);
|
||||
bool request_gpu_as_optional = false);
|
||||
|
||||
// This method can be called from FillExpectations to set the correct types
|
||||
// for the shared GL input side packet(s).
|
||||
|
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
#include "mediapipe/gpu/gl_texture_buffer.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
@ -131,6 +133,13 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
|
|||
SymbolAvailable(&glTexStorage2D)) {
|
||||
ABSL_CHECK(data == nullptr) << "unimplemented";
|
||||
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
|
||||
} else if (info.immutable) {
|
||||
ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
|
||||
context->GetGlVersion() != GlVersion::kGLES2)
|
||||
<< "Immutable GpuBuffer format requested is not supported in this "
|
||||
<< "GlContext. Format was " << static_cast<uint32_t>(format_);
|
||||
ABSL_CHECK(data == nullptr) << "unimplemented";
|
||||
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
|
||||
} else {
|
||||
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
|
||||
height_, 0 /* border */, info.gl_format, info.gl_type, data);
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
#include <cstdlib>
|
||||
|
||||
#if defined(MEDIAPIPE_USING_SWIFTSHADER)
|
||||
#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
|
||||
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
|
||||
#endif
|
||||
|
||||
|
|
|
@ -35,6 +35,10 @@ namespace mediapipe {
|
|||
#endif // GL_HALF_FLOAT_OES
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
#ifndef GL_RGBA8
|
||||
#define GL_RGBA8 0x8058
|
||||
#endif // GL_RGBA8
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#ifdef GL_ES_VERSION_2_0
|
||||
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
|
||||
|
@ -163,6 +167,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
|||
{
|
||||
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
|
||||
}},
|
||||
{GpuBufferFormat::kImmutableRGBAFloat128,
|
||||
{
|
||||
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
|
||||
}},
|
||||
{GpuBufferFormat::kImmutableRGBA32,
|
||||
{
|
||||
{GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
|
||||
}},
|
||||
}};
|
||||
|
||||
static const auto* gles2_format_info = ([] {
|
||||
|
@ -206,6 +218,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
|||
|
||||
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
|
||||
switch (format) {
|
||||
case GpuBufferFormat::kImmutableRGBA32:
|
||||
case GpuBufferFormat::kBGRA32:
|
||||
// TODO: verify we are handling order of channels correctly.
|
||||
return ImageFormat::SRGBA;
|
||||
|
@ -221,10 +234,11 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
|
|||
return ImageFormat::SRGB;
|
||||
case GpuBufferFormat::kTwoComponentFloat32:
|
||||
return ImageFormat::VEC32F2;
|
||||
case GpuBufferFormat::kImmutableRGBAFloat128:
|
||||
case GpuBufferFormat::kRGBAFloat128:
|
||||
return ImageFormat::VEC32F4;
|
||||
case GpuBufferFormat::kRGBA32:
|
||||
// TODO: this likely maps to ImageFormat::SRGBA
|
||||
return ImageFormat::SRGBA;
|
||||
case GpuBufferFormat::kGrayHalf16:
|
||||
case GpuBufferFormat::kOneComponent8Alpha:
|
||||
case GpuBufferFormat::kOneComponent8Red:
|
||||
|
|
|
@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t {
|
|||
kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible.
|
||||
kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'),
|
||||
kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
|
||||
// Immutable version of kRGBA32
|
||||
kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'),
|
||||
// Immutable version of kRGBAFloat128
|
||||
kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'),
|
||||
// 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling.
|
||||
kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'),
|
||||
// 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling.
|
||||
|
@ -78,6 +82,9 @@ struct GlTextureInfo {
|
|||
// For multiplane buffers, this represents how many times smaller than
|
||||
// the nominal image size a plane is.
|
||||
int downscale;
|
||||
// For GLES3.1+ compute shaders, users may explicitly request immutable
|
||||
// textures.
|
||||
bool immutable = false;
|
||||
};
|
||||
|
||||
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
||||
|
@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
|
|||
return kCVPixelFormatType_64RGBAHalf;
|
||||
case GpuBufferFormat::kRGBAFloat128:
|
||||
return kCVPixelFormatType_128RGBAFloat;
|
||||
case GpuBufferFormat::kImmutableRGBA32:
|
||||
case GpuBufferFormat::kImmutableRGBAFloat128:
|
||||
case GpuBufferFormat::kNV12:
|
||||
case GpuBufferFormat::kNV21:
|
||||
case GpuBufferFormat::kI420:
|
||||
|
|
|
@ -151,7 +151,7 @@ static std::shared_ptr<GpuBufferStorageCvPixelBuffer> ConvertFromImageFrame(
|
|||
std::shared_ptr<GpuBufferStorageImageFrame> frame) {
|
||||
auto status_or_buffer =
|
||||
CreateCVPixelBufferForImageFrame(frame->image_frame());
|
||||
ABSL_CHECK(status_or_buffer.ok());
|
||||
ABSL_CHECK_OK(status_or_buffer);
|
||||
return std::make_shared<GpuBufferStorageCvPixelBuffer>(
|
||||
std::move(status_or_buffer).value());
|
||||
}
|
||||
|
|
|
@ -16,6 +16,9 @@ syntax = "proto3";
|
|||
|
||||
package mediapipe;
|
||||
|
||||
option java_package = "com.google.mediapipe.gpu";
|
||||
option java_outer_classname = "GpuOriginProto";
|
||||
|
||||
message GpuOrigin {
|
||||
enum Mode {
|
||||
DEFAULT = 0;
|
||||
|
|
|
@ -14,10 +14,14 @@
|
|||
|
||||
#include "mediapipe/gpu/gpu_shared_data_internal.h"
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/base/attributes.h"
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/deps/no_destructor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/gpu/gl_context.h"
|
||||
#include "mediapipe/gpu/gl_context_options.pb.h"
|
||||
#include "mediapipe/gpu/graph_support.h"
|
||||
|
@ -83,8 +87,25 @@ GpuResources::StatusOrGpuResources GpuResources::Create(
|
|||
}
|
||||
|
||||
GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
|
||||
: gl_key_context_(new GlContextMapType(),
|
||||
[](auto* map) {
|
||||
// This flushes all pending jobs in all GL contexts,
|
||||
// ensuring that all GL contexts not referenced
|
||||
// elsewhere are destroyed as part of this destructor.
|
||||
// Failure to do this may cause GL threads to outlast
|
||||
// this destructor and execute jobs after the
|
||||
// GpuResources object is destroyed.
|
||||
for (auto& [key, context] : *map) {
|
||||
const auto status = std::move(context)->Run(
|
||||
[]() { return absl::OkStatus(); });
|
||||
ABSL_LOG_IF(ERROR, !status.ok())
|
||||
<< "Failed to flush GlContext jobs: " << status;
|
||||
}
|
||||
delete map;
|
||||
})
|
||||
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
: texture_caches_(std::make_shared<CvTextureCacheManager>()),
|
||||
,
|
||||
texture_caches_(std::make_shared<CvTextureCacheManager>()),
|
||||
gpu_buffer_pool_(
|
||||
[tc = texture_caches_](const internal::GpuBufferSpec& spec,
|
||||
const MultiPoolOptions& options) {
|
||||
|
@ -92,7 +113,7 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
|
|||
})
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
{
|
||||
gl_key_context_[SharedContextKey()] = gl_context;
|
||||
gl_key_context_->insert({SharedContextKey(), gl_context});
|
||||
named_executors_[kGpuExecutorName] =
|
||||
std::make_shared<GlContextExecutor>(gl_context.get());
|
||||
#if __APPLE__
|
||||
|
@ -104,6 +125,15 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
|
|||
}
|
||||
|
||||
GpuResources::~GpuResources() {
|
||||
// This flushes all pending jobs in all GL contexts,
|
||||
// ensuring that all existing jobs, which may refer GpuResource and kept their
|
||||
// gpu resources (e.g. GpuResources::gpu_buffer_pool_) through a raw pointer,
|
||||
// have finished before kept gpu resources get deleted.
|
||||
for (auto& [key, context] : *gl_key_context_) {
|
||||
const auto status = context->Run([]() { return absl::OkStatus(); });
|
||||
ABSL_LOG_IF(ERROR, !status.ok())
|
||||
<< "Failed to flush GlContext jobs: " << status;
|
||||
}
|
||||
#if __APPLE__
|
||||
// Note: on Apple platforms, this object contains Objective-C objects.
|
||||
// The destructor will release them, but ARC must be on.
|
||||
|
@ -111,7 +141,7 @@ GpuResources::~GpuResources() {
|
|||
#error This file must be built with ARC.
|
||||
#endif
|
||||
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
for (auto& kv : gl_key_context_) {
|
||||
for (auto& kv : *gl_key_context_) {
|
||||
texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache());
|
||||
}
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
|
@ -173,23 +203,24 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
|
|||
const std::shared_ptr<GlContext>& GpuResources::gl_context(
|
||||
CalculatorContext* cc) {
|
||||
if (cc) {
|
||||
auto it = gl_key_context_.find(node_key_[cc->NodeName()]);
|
||||
if (it != gl_key_context_.end()) {
|
||||
auto it = gl_key_context_->find(node_key_[cc->NodeName()]);
|
||||
if (it != gl_key_context_->end()) {
|
||||
return it->second;
|
||||
}
|
||||
}
|
||||
|
||||
return gl_key_context_[SharedContextKey()];
|
||||
return gl_key_context_->at(SharedContextKey());
|
||||
}
|
||||
|
||||
GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext(
|
||||
const std::string& key) {
|
||||
auto it = gl_key_context_.find(key);
|
||||
if (it == gl_key_context_.end()) {
|
||||
MP_ASSIGN_OR_RETURN(std::shared_ptr<GlContext> new_context,
|
||||
GlContext::Create(*gl_key_context_[SharedContextKey()],
|
||||
kGlContextUseDedicatedThread));
|
||||
it = gl_key_context_.emplace(key, new_context).first;
|
||||
auto it = gl_key_context_->find(key);
|
||||
if (it == gl_key_context_->end()) {
|
||||
MP_ASSIGN_OR_RETURN(
|
||||
std::shared_ptr<GlContext> new_context,
|
||||
GlContext::Create(*gl_key_context_->at(SharedContextKey()),
|
||||
kGlContextUseDedicatedThread));
|
||||
it = gl_key_context_->emplace(key, new_context).first;
|
||||
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
texture_caches_->RegisterTextureCache(it->second->cv_texture_cache());
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
|
|
|
@ -21,6 +21,8 @@
|
|||
#ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
|
||||
#define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_node.h"
|
||||
#include "mediapipe/framework/executor.h"
|
||||
|
@ -82,7 +84,10 @@ class GpuResources {
|
|||
const std::string& ContextKey(const std::string& canonical_node_name);
|
||||
|
||||
std::map<std::string, std::string> node_key_;
|
||||
std::map<std::string, std::shared_ptr<GlContext>> gl_key_context_;
|
||||
|
||||
using GlContextMapType = std::map<std::string, std::shared_ptr<GlContext>>;
|
||||
std::unique_ptr<GlContextMapType, void (*)(GlContextMapType*)>
|
||||
gl_key_context_;
|
||||
|
||||
#ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
||||
std::shared_ptr<CvTextureCacheManager> texture_caches_;
|
||||
|
|
|
@ -15,6 +15,9 @@
|
|||
#ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_
|
||||
#define MEDIAPIPE_GPU_GPU_TEST_BASE_H_
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
|
@ -22,9 +25,9 @@
|
|||
|
||||
namespace mediapipe {
|
||||
|
||||
class GpuTestBase : public ::testing::Test {
|
||||
class GpuTestEnvironment {
|
||||
protected:
|
||||
GpuTestBase() { helper_.InitializeForTest(gpu_resources_.get()); }
|
||||
GpuTestEnvironment() { helper_.InitializeForTest(gpu_resources_.get()); }
|
||||
|
||||
void RunInGlContext(std::function<void(void)> gl_func) {
|
||||
helper_.RunInGlContext(std::move(gl_func));
|
||||
|
@ -35,6 +38,12 @@ class GpuTestBase : public ::testing::Test {
|
|||
GlCalculatorHelper helper_;
|
||||
};
|
||||
|
||||
class GpuTestBase : public testing::Test, public GpuTestEnvironment {};
|
||||
|
||||
template <typename T>
|
||||
class GpuTestWithParamBase : public testing::TestWithParam<T>,
|
||||
public GpuTestEnvironment {};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GPU_GPU_TEST_BASE_H_
|
||||
|
|
|
@ -171,7 +171,7 @@ METAL_LIBRARY_ATTRS = dicts.add(apple_support.action_required_attrs(), {
|
|||
metal_library = rule(
|
||||
implementation = _metal_library_impl,
|
||||
attrs = METAL_LIBRARY_ATTRS,
|
||||
fragments = ["apple", "objc", "swift"],
|
||||
fragments = ["apple", "objc"],
|
||||
output_to_genfiles = True,
|
||||
)
|
||||
"""
|
||||
|
|
|
@ -50,9 +50,10 @@
|
|||
- (CVMetalTextureCacheRef)mtlTextureCache {
|
||||
@synchronized(self) {
|
||||
if (!_mtlTextureCache) {
|
||||
CVReturn __unused err =
|
||||
CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
|
||||
NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d ; device %@", err,
|
||||
CVReturn __unused err = CVMetalTextureCacheCreate(
|
||||
NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
|
||||
NSAssert(err == kCVReturnSuccess,
|
||||
@"Error at CVMetalTextureCacheCreate %d ; device %@", err,
|
||||
self.mtlDevice);
|
||||
// TODO: register and flush metal caches too.
|
||||
}
|
|
@ -24,23 +24,27 @@
|
|||
|
||||
namespace mediapipe {
|
||||
|
||||
CVPixelBufferPoolRef CreateCVPixelBufferPool(
|
||||
int width, int height, OSType pixelFormat, int keepCount,
|
||||
CFTimeInterval maxAge) {
|
||||
CVPixelBufferPoolRef CreateCVPixelBufferPool(int width, int height,
|
||||
OSType pixelFormat, int keepCount,
|
||||
CFTimeInterval maxAge) {
|
||||
CVPixelBufferPoolRef pool = NULL;
|
||||
|
||||
NSMutableDictionary *sourcePixelBufferOptions =
|
||||
[(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy];
|
||||
[(__bridge NSDictionary *)GetCVPixelBufferAttributesForGlCompatibility()
|
||||
mutableCopy];
|
||||
[sourcePixelBufferOptions addEntriesFromDictionary:@{
|
||||
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
|
||||
(id)kCVPixelBufferWidthKey : @(width),
|
||||
(id)kCVPixelBufferHeightKey : @(height),
|
||||
}];
|
||||
|
||||
NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init];
|
||||
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount);
|
||||
NSMutableDictionary *pixelBufferPoolOptions =
|
||||
[[NSMutableDictionary alloc] init];
|
||||
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] =
|
||||
@(keepCount);
|
||||
if (maxAge > 0) {
|
||||
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = @(maxAge);
|
||||
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] =
|
||||
@(maxAge);
|
||||
}
|
||||
|
||||
CVPixelBufferPoolCreate(
|
||||
|
@ -50,8 +54,9 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
|
|||
return pool;
|
||||
}
|
||||
|
||||
OSStatus PreallocateCVPixelBufferPoolBuffers(
|
||||
CVPixelBufferPoolRef pool, int count, CFDictionaryRef auxAttributes) {
|
||||
OSStatus PreallocateCVPixelBufferPoolBuffers(CVPixelBufferPoolRef pool,
|
||||
int count,
|
||||
CFDictionaryRef auxAttributes) {
|
||||
CVReturn err = kCVReturnSuccess;
|
||||
NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init];
|
||||
for (int i = 0; i < count && err == kCVReturnSuccess; i++) {
|
||||
|
@ -68,30 +73,37 @@ OSStatus PreallocateCVPixelBufferPoolBuffers(
|
|||
return err;
|
||||
}
|
||||
|
||||
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(int allocationThreshold) {
|
||||
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(
|
||||
int allocationThreshold) {
|
||||
if (allocationThreshold > 0) {
|
||||
return (CFDictionaryRef)CFBridgingRetain(
|
||||
@{(id)kCVPixelBufferPoolAllocationThresholdKey: @(allocationThreshold)});
|
||||
return (CFDictionaryRef)CFBridgingRetain(@{
|
||||
(id)kCVPixelBufferPoolAllocationThresholdKey : @(allocationThreshold)
|
||||
});
|
||||
} else {
|
||||
return nil;
|
||||
}
|
||||
}
|
||||
|
||||
CVReturn CreateCVPixelBufferWithPool(
|
||||
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
|
||||
CVTextureCacheType textureCache, CVPixelBufferRef* outBuffer) {
|
||||
return CreateCVPixelBufferWithPool(pool, auxAttributes, [textureCache](){
|
||||
CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
|
||||
CFDictionaryRef auxAttributes,
|
||||
CVTextureCacheType textureCache,
|
||||
CVPixelBufferRef *outBuffer) {
|
||||
return CreateCVPixelBufferWithPool(
|
||||
pool, auxAttributes,
|
||||
[textureCache]() {
|
||||
#if TARGET_OS_OSX
|
||||
CVOpenGLTextureCacheFlush(textureCache, 0);
|
||||
CVOpenGLTextureCacheFlush(textureCache, 0);
|
||||
#else
|
||||
CVOpenGLESTextureCacheFlush(textureCache, 0);
|
||||
CVOpenGLESTextureCacheFlush(textureCache, 0);
|
||||
#endif // TARGET_OS_OSX
|
||||
}, outBuffer);
|
||||
},
|
||||
outBuffer);
|
||||
}
|
||||
|
||||
CVReturn CreateCVPixelBufferWithPool(
|
||||
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
|
||||
std::function<void(void)> flush, CVPixelBufferRef* outBuffer) {
|
||||
CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
|
||||
CFDictionaryRef auxAttributes,
|
||||
std::function<void(void)> flush,
|
||||
CVPixelBufferRef *outBuffer) {
|
||||
CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(
|
||||
kCFAllocatorDefault, pool, auxAttributes, outBuffer);
|
||||
if (err == kCVReturnWouldExceedAllocationThreshold) {
|
||||
|
@ -103,11 +115,13 @@ CVReturn CreateCVPixelBufferWithPool(
|
|||
kCFAllocatorDefault, pool, auxAttributes, outBuffer);
|
||||
}
|
||||
if (err == kCVReturnWouldExceedAllocationThreshold) {
|
||||
// TODO: allow the application to set the threshold. For now, disable it by
|
||||
// default, since the threshold we are using is arbitrary and some graphs routinely cross it.
|
||||
// TODO: allow the application to set the threshold. For now, disable it
|
||||
// by default, since the threshold we are using is arbitrary and some
|
||||
// graphs routinely cross it.
|
||||
#ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK
|
||||
NSLog(@"Using more buffers than expected! This is a debug-only warning, "
|
||||
"you can ignore it if your app works fine otherwise.");
|
||||
NSLog(
|
||||
@"Using more buffers than expected! This is a debug-only warning, "
|
||||
"you can ignore it if your app works fine otherwise.");
|
||||
#ifdef DEBUG
|
||||
NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description);
|
||||
#endif // DEBUG
|
|
@ -17,7 +17,6 @@ package com.google.mediapipe.components;
|
|||
import static java.lang.Math.max;
|
||||
import static java.lang.Math.min;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.SurfaceTexture;
|
||||
import android.opengl.GLES11Ext;
|
||||
import android.opengl.GLES20;
|
||||
|
@ -25,6 +24,7 @@ import android.opengl.GLES31;
|
|||
import android.opengl.GLSurfaceView;
|
||||
import android.opengl.Matrix;
|
||||
import android.util.Log;
|
||||
import android.util.Pair;
|
||||
import com.google.mediapipe.framework.TextureFrame;
|
||||
import com.google.mediapipe.glutil.CommonShaders;
|
||||
import com.google.mediapipe.glutil.ShaderUtil;
|
||||
|
@ -51,11 +51,9 @@ import javax.microedition.khronos.opengles.GL10;
|
|||
* {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}).
|
||||
*/
|
||||
public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
||||
/**
|
||||
* Listener for Bitmap capture requests.
|
||||
*/
|
||||
public interface BitmapCaptureListener {
|
||||
void onBitmapCaptured(Bitmap result);
|
||||
/** Listener for image capture requests. */
|
||||
public interface ImageCaptureListener {
|
||||
void onImageCaptured(int width, int height, int[] data);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -86,27 +84,28 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
private float[] textureTransformMatrix = new float[16];
|
||||
private SurfaceTexture surfaceTexture = null;
|
||||
private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
|
||||
private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean();
|
||||
private BitmapCaptureListener bitmapCaptureListener;
|
||||
private final AtomicBoolean captureNextFrameImage = new AtomicBoolean();
|
||||
private ImageCaptureListener imageCaptureListener;
|
||||
// Specifies whether a black CLAMP_TO_BORDER effect should be used.
|
||||
private boolean shouldClampToBorder = false;
|
||||
private Scale scale = Scale.FILL;
|
||||
|
||||
/**
|
||||
* Sets the {@link BitmapCaptureListener}.
|
||||
*/
|
||||
public void setBitmapCaptureListener(BitmapCaptureListener bitmapCaptureListener) {
|
||||
this.bitmapCaptureListener = bitmapCaptureListener;
|
||||
private float zoomFactor = 1.0f;
|
||||
private Pair<Float, Float> zoomLocation = new Pair<>(0.5f, 0.5f);
|
||||
|
||||
/** Sets the {@link ImageCaptureListener}. */
|
||||
public void setImageCaptureListener(ImageCaptureListener imageCaptureListener) {
|
||||
this.imageCaptureListener = imageCaptureListener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Request to capture Bitmap of the next frame.
|
||||
* Request to capture Image of the next frame.
|
||||
*
|
||||
* The result will be provided to the {@link BitmapCaptureListener} if one is set. Please note
|
||||
* <p>The result will be provided to the {@link ImageCaptureListener} if one is set. Please note
|
||||
* this is an expensive operation and the result may not be available for a while.
|
||||
*/
|
||||
public void captureNextFrameBitmap() {
|
||||
captureNextFrameBitmap.set(true);
|
||||
public void captureNextFrameImage() {
|
||||
captureNextFrameImage.set(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -202,9 +201,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
|
||||
ShaderUtil.checkGlError("glDrawArrays");
|
||||
|
||||
// Capture Bitmap if requested.
|
||||
BitmapCaptureListener bitmapCaptureListener = this.bitmapCaptureListener;
|
||||
if (captureNextFrameBitmap.getAndSet(false) && bitmapCaptureListener != null) {
|
||||
// Capture image if requested.
|
||||
ImageCaptureListener imageCaptureListener = this.imageCaptureListener;
|
||||
if (captureNextFrameImage.getAndSet(false) && imageCaptureListener != null) {
|
||||
// Find the name of the bound texture.
|
||||
int[] texName = new int[1];
|
||||
if (surfaceTexture != null) {
|
||||
|
@ -219,8 +218,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1);
|
||||
int texWidth = texDims[0];
|
||||
int texHeight = texDims[1];
|
||||
int bitmapSize = texWidth * texHeight;
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(bitmapSize * 4);
|
||||
int imageSize = texWidth * texHeight;
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(imageSize * 4);
|
||||
byteBuffer.order(ByteOrder.nativeOrder());
|
||||
|
||||
// Read pixels from texture.
|
||||
|
@ -235,27 +234,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
GLES20.glDeleteFramebuffers(1, fbo, 0);
|
||||
ShaderUtil.checkGlError("capture frame");
|
||||
|
||||
int[] pixelBuffer = new int[bitmapSize];
|
||||
byteBuffer.asIntBuffer().get(pixelBuffer);
|
||||
for (int i = 0; i < bitmapSize; i++) {
|
||||
// Swap R and B channels.
|
||||
pixelBuffer[i] =
|
||||
(pixelBuffer[i] & 0xff00ff00)
|
||||
| ((pixelBuffer[i] & 0x000000ff) << 16)
|
||||
| ((pixelBuffer[i] & 0x00ff0000) >> 16);
|
||||
}
|
||||
|
||||
// Send bitmap.
|
||||
Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888);
|
||||
bitmap.setPixels(
|
||||
pixelBuffer,
|
||||
/* offset= */ bitmapSize - texWidth,
|
||||
/* stride= */ -texWidth,
|
||||
/* x= */ 0,
|
||||
/* y= */ 0,
|
||||
texWidth,
|
||||
texHeight);
|
||||
bitmapCaptureListener.onBitmapCaptured(bitmap);
|
||||
int[] data = new int[imageSize];
|
||||
byteBuffer.asIntBuffer().get(data);
|
||||
imageCaptureListener.onImageCaptured(texWidth, texHeight, data);
|
||||
}
|
||||
|
||||
GLES20.glBindTexture(textureTarget, 0);
|
||||
|
@ -294,6 +275,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
float textureBottom = (1.0f - scaleHeight) * alignmentVertical;
|
||||
float textureTop = textureBottom + scaleHeight;
|
||||
|
||||
Pair<Float, Float> currentZoomLocation = this.zoomLocation;
|
||||
float zoomLocationX = currentZoomLocation.first;
|
||||
float zoomLocationY = currentZoomLocation.second;
|
||||
|
||||
textureLeft = (textureLeft - 0.5f) / zoomFactor + zoomLocationX;
|
||||
textureRight = (textureRight - 0.5f) / zoomFactor + zoomLocationX;
|
||||
textureBottom = (textureBottom - 0.5f) / zoomFactor + zoomLocationY;
|
||||
textureTop = (textureTop - 0.5f) / zoomFactor + zoomLocationY;
|
||||
|
||||
return new float[] {textureLeft, textureRight, textureBottom, textureTop};
|
||||
}
|
||||
|
||||
|
@ -380,6 +370,22 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
|
|||
this.scale = scale;
|
||||
}
|
||||
|
||||
/** Zoom factor applied to the frame, must not be 0. */
|
||||
public void setZoomFactor(float zoomFactor) {
|
||||
if (zoomFactor == 0.f) {
|
||||
return;
|
||||
}
|
||||
this.zoomFactor = zoomFactor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Location where to apply the zooming of the frame to. Default is 0.5, 0.5 (scaling is applied to
|
||||
* the center).
|
||||
*/
|
||||
public void setZoomLocation(float zoomLocationX, float zoomLocationY) {
|
||||
this.zoomLocation = new Pair<>(zoomLocationX, zoomLocationY);
|
||||
}
|
||||
|
||||
private boolean isExternalTexture() {
|
||||
return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
|
||||
}
|
||||
|
|
|
@ -128,6 +128,16 @@ public final class PacketGetter {
|
|||
return ProtoUtil.unpack(result, defaultInstance);
|
||||
}
|
||||
|
||||
public static <T extends MessageLite> T getProto(final Packet packet, Parser<T> messageParser) {
|
||||
SerializedMessage result = new SerializedMessage();
|
||||
nativeGetProto(packet.getNativeHandle(), result);
|
||||
try {
|
||||
return messageParser.parseFrom(result.value);
|
||||
} catch (InvalidProtocolBufferException e) {
|
||||
throw new IllegalArgumentException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds.
|
||||
*/
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user