cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed function names in MPPImage Utils

Browse Source
This commit is contained in:
Prianka Liz Kariat 2023-12-11 19:40:13 +05:30
commit 04f826e9d3
381 changed files with 26561 additions and 1515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bazelrc
View File

@ -98,6 +98,9 @@ build:darwin_arm64 --apple_platform_type=macos
build:darwin_arm64 --macos_minimum_os=10.16 build:darwin_arm64 --macos_minimum_os=10.16
build:darwin_arm64 --cpu=darwin_arm64 build:darwin_arm64 --cpu=darwin_arm64
# Turn off maximum stdout size
build --experimental_ui_max_stdouterr_bytes=-1
# This bazelrc file is meant to be written by a setup script. # This bazelrc file is meant to be written by a setup script.
try-import %workspace%/.configure.bazelrc try-import %workspace%/.configure.bazelrc

3
WORKSPACE
View File

@ -513,6 +513,9 @@ http_archive(
"@//third_party:org_tensorflow_system_python.diff", "@//third_party:org_tensorflow_system_python.diff",
# Diff is generated with a script, don't update it manually. # Diff is generated with a script, don't update it manually.
"@//third_party:org_tensorflow_custom_ops.diff", "@//third_party:org_tensorflow_custom_ops.diff",
# Works around Bazel issue with objc_library.
# See https://github.com/bazelbuild/bazel/issues/19912
"@//third_party:org_tensorflow_objc_build_fixes.diff",
], ],
patch_args = [ patch_args = [
"-p1", "-p1",

342
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj Normal file
View File

@ -0,0 +1,342 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 56;
objects = {
/* Begin PBXBuildFile section */
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; };
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
8566B5562ABABF9A00AAB22A /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
8566B54F2ABABF9A00AAB22A = {
isa = PBXGroup;
children = (
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
8566B55A2ABABF9A00AAB22A /* Products */,
);
sourceTree = "<group>";
};
8566B55A2ABABF9A00AAB22A /* Products */ = {
isa = PBXGroup;
children = (
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */,
);
name = Products;
sourceTree = "<group>";
};
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
isa = PBXGroup;
children = (
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */,
);
path = MediaPipeTasksDocGen;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXHeadersBuildPhase section */
8566B5542ABABF9A00AAB22A /* Headers */ = {
isa = PBXHeadersBuildPhase;
buildActionMask = 2147483647;
files = (
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXHeadersBuildPhase section */
/* Begin PBXNativeTarget section */
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
isa = PBXNativeTarget;
buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */;
buildPhases = (
8566B5542ABABF9A00AAB22A /* Headers */,
8566B5552ABABF9A00AAB22A /* Sources */,
8566B5562ABABF9A00AAB22A /* Frameworks */,
8566B5572ABABF9A00AAB22A /* Resources */,
);
buildRules = (
);
dependencies = (
);
name = MediaPipeTasksDocGen;
productName = MediaPipeTasksDocGen;
productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */;
productType = "com.apple.product-type.framework";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
8566B5502ABABF9A00AAB22A /* Project object */ = {
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = 1;
LastUpgradeCheck = 1430;
TargetAttributes = {
8566B5582ABABF9A00AAB22A = {
CreatedOnToolsVersion = 14.3.1;
};
};
};
buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */;
compatibilityVersion = "Xcode 14.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = 8566B54F2ABABF9A00AAB22A;
productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
8566B5572ABABF9A00AAB22A /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
8566B5552ABABF9A00AAB22A /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin XCBuildConfiguration section */
8566B55E2ABABF9A00AAB22A /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
VERSIONING_SYSTEM = "apple-generic";
VERSION_INFO_PREFIX = "";
};
name = Debug;
};
8566B55F2ABABF9A00AAB22A /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
MTL_ENABLE_DEBUG_INFO = NO;
MTL_FAST_MATH = YES;
SDKROOT = iphoneos;
SWIFT_COMPILATION_MODE = wholemodule;
SWIFT_OPTIMIZATION_LEVEL = "-O";
VALIDATE_PRODUCT = YES;
VERSIONING_SYSTEM = "apple-generic";
VERSION_INFO_PREFIX = "";
};
name = Release;
};
8566B5612ABABF9A00AAB22A /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEFINES_MODULE = YES;
DYLIB_COMPATIBILITY_VERSION = 1;
DYLIB_CURRENT_VERSION = 1;
DYLIB_INSTALL_NAME_BASE = "@rpath";
ENABLE_MODULE_VERIFIER = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
8566B5622ABABF9A00AAB22A /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEFINES_MODULE = YES;
DYLIB_COMPATIBILITY_VERSION = 1;
DYLIB_CURRENT_VERSION = 1;
DYLIB_INSTALL_NAME_BASE = "@rpath";
ENABLE_MODULE_VERIFIER = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = {
isa = XCConfigurationList;
buildConfigurations = (
8566B55E2ABABF9A00AAB22A /* Debug */,
8566B55F2ABABF9A00AAB22A /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = {
isa = XCConfigurationList;
buildConfigurations = (
8566B5612ABABF9A00AAB22A /* Debug */,
8566B5622ABABF9A00AAB22A /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
};
rootObject = 8566B5502ABABF9A00AAB22A /* Project object */;
}

7
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata generated Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:">
</FileRef>
</Workspace>

8
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>

BIN
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcuserdata/macd.xcuserdatad/UserInterfaceState.xcuserstate generated Normal file
View File

Binary file not shown.

14
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/xcuserdata/macd.xcuserdatad/xcschemes/xcschememanagement.plist Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>MediaPipeTasksDocGen.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>

17
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h Normal file
View File

@ -0,0 +1,17 @@
//
// MediaPipeTasksDocGen.h
// MediaPipeTasksDocGen
//
// Created by Mark McDonald on 20/9/2023.
//
#import <Foundation/Foundation.h>
//! Project version number for MediaPipeTasksDocGen.
FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber;
//! Project version string for MediaPipeTasksDocGen.
FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[];
// In this header, you should import all the public headers of your framework using statements like
// #import <MediaPipeTasksDocGen/PublicHeader.h>

11
docs/MediaPipeTasksDocGen/Podfile Normal file
View File

@ -0,0 +1,11 @@
# Uncomment the next line to define a global platform for your project
platform :ios, '15.0'
target 'MediaPipeTasksDocGen' do
# Comment the next line if you don't want to use dynamic frameworks
use_frameworks!
# Pods for MediaPipeTasksDocGen
pod 'MediaPipeTasksText'
pod 'MediaPipeTasksVision'
end

9
docs/MediaPipeTasksDocGen/README.md Normal file
View File

@ -0,0 +1,9 @@
# MediaPipeTasksDocGen
This empty project is used to generate reference documentation for the
ObjectiveC and Swift libraries.
Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published
to [the developer site](https://developers.google.com/mediapipe/solutions/).
To bump the API version used, edit [`Podfile`](./Podfile).

2
mediapipe/calculators/audio/spectrogram_calculator.proto
View File

@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on // If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input // the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on // packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the intial input // the cumulative timestamping, which is inferred from the initial input
// timestamp and the cumulative number of samples. // timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 8 [default = false]; optional bool use_local_timestamp = 8 [default = false];
} }

2
mediapipe/calculators/audio/time_series_framer_calculator.proto
View File

@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on // If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input // the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on // packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the intial input // the cumulative timestamping, which is inferred from the initial input
// timestamp and the cumulative number of samples. // timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 6 [default = false]; optional bool use_local_timestamp = 6 [default = false];
} }

2
mediapipe/calculators/core/BUILD
View File

@ -727,6 +727,7 @@ cc_library(
"//mediapipe/framework/port:logging", "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
], ],
alwayslink = 1, alwayslink = 1,
) )
@ -742,6 +743,7 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util", "//mediapipe/framework/tool:options_util",
"//mediapipe/util:packet_test_util",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],

2
mediapipe/calculators/core/packet_sequencer_calculator_test.cc
View File

@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator")); CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
} }
// Shows how control packets recieve timestamps before and after frame packets // Shows how control packets receive timestamps before and after frame packets
// have arrived. // have arrived.
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) { TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
CalculatorGraphConfig::Node node_config = BuildNodeConfig(); CalculatorGraphConfig::Node node_config = BuildNodeConfig();

35
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
View File

@ -17,6 +17,7 @@
#include <set> #include <set>
#include <string> #include <string>
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@ -32,6 +33,7 @@ namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM"; constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM"; constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO"; constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
constexpr char kTagAtTick[] = "AT_TICK"; constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK"; constexpr char kTagTick[] = "TICK";
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP"; constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
@ -43,6 +45,7 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
res->emplace(kTagAtPostStream, Timestamp::PostStream()); res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0)); res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset()); res->emplace(kTagAtTick, Timestamp::Unset());
res->emplace(kTagAtFirstTick, Timestamp::Unset());
res->emplace(kTagAtTimestamp, Timestamp::Unset()); res->emplace(kTagAtTimestamp, Timestamp::Unset());
return res; return res;
}(); }();
@ -59,8 +62,8 @@ std::string GetOutputTag(const CC& cc) {
// timestamp, depending on the tag used to define output stream(s). (One tag can // timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.) // be used only.)
// //
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP // Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
// and corresponding timestamps are Timestamp::PreStream(), // AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK // Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
// input, and timestamp received from a side input. // input, and timestamp received from a side input.
// //
@ -96,6 +99,7 @@ class SidePacketToStreamCalculator : public CalculatorBase {
private: private:
bool is_tick_processing_ = false; bool is_tick_processing_ = false;
bool close_on_first_tick_ = false;
std::string output_tag_; std::string output_tag_;
}; };
REGISTER_CALCULATOR(SidePacketToStreamCalculator); REGISTER_CALCULATOR(SidePacketToStreamCalculator);
@ -103,13 +107,16 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) { absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
const auto& tags = cc->Outputs().GetTags(); const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1) RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
"AT_TIMESTAMP tags is allowed and required to specify output " "AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
"stream(s)."; "specify output stream(s).";
RET_CHECK( const bool has_tick_output =
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) || cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick))) const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
<< "Either both of TICK and AT_TICK should be used or none of them."; RET_CHECK((has_tick_output && has_tick_input) ||
(!has_tick_output && !has_tick_input))
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them.";
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) && RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) || cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
(!cc->Outputs().HasTag(kTagAtTimestamp) && (!cc->Outputs().HasTag(kTagAtTimestamp) &&
@ -148,11 +155,17 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
// timestamp bound update. // timestamp bound update.
cc->SetOffset(TimestampDiff(0)); cc->SetOffset(TimestampDiff(0));
} }
if (output_tag_ == kTagAtFirstTick) {
close_on_first_tick_ = true;
}
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) { absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
if (is_tick_processing_) { if (is_tick_processing_) {
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
return absl::OkStatus();
}
// TICK input is guaranteed to be non-empty, as it's the only input stream // TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator. // for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp(); const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
@ -160,6 +173,9 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
cc->Outputs() cc->Outputs()
.Get(output_tag_, i) .Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); .AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
if (close_on_first_tick_) {
cc->Outputs().Get(output_tag_, i).Close();
}
} }
return absl::OkStatus(); return absl::OkStatus();
@ -170,6 +186,7 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) { absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
if (!cc->Outputs().HasTag(kTagAtTick) && if (!cc->Outputs().HasTag(kTagAtTick) &&
!cc->Outputs().HasTag(kTagAtFirstTick) &&
!cc->Outputs().HasTag(kTagAtTimestamp)) { !cc->Outputs().HasTag(kTagAtTimestamp)) {
const auto& timestamp = kTimestampMap->at(output_tag_); const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {

166
mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc
View File

@ -27,13 +27,17 @@
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h" #include "mediapipe/framework/tool/options_util.h"
#include "mediapipe/util/packet_test_util.h"
namespace mediapipe { namespace mediapipe {
namespace { namespace {
using testing::HasSubstr; using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::HasSubstr;
using ::testing::IsEmpty;
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) { TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -52,10 +56,35 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
EXPECT_THAT( EXPECT_THAT(
status.message(), status.message(),
HasSubstr( HasSubstr(
"Either both of TICK and AT_TICK should be used or none of them.")); "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
} }
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) { TEST(SidePacketToStreamCalculator,
WrongConfigWithMissingTickForFirstTickProcessing) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -76,7 +105,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
"or none of them.")); "or none of them."));
} }
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) { TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -92,14 +121,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraph graph; CalculatorGraph graph;
auto status = graph.Initialize(graph_config); auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok()); EXPECT_FALSE(status.ok());
EXPECT_THAT( EXPECT_THAT(status.message(),
status.message(), HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"AT_TIMESTAMP tags is allowed and required to specify output " "allowed and required to specify output stream(s)."));
"stream(s)."));
} }
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) { TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -117,14 +145,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraph graph; CalculatorGraph graph;
auto status = graph.Initialize(graph_config); auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok()); EXPECT_FALSE(status.ok());
EXPECT_THAT( EXPECT_THAT(status.message(),
status.message(), HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"AT_TIMESTAMP tags is allowed and required to specify output " "allowed and required to specify output stream(s)."));
"stream(s)."));
} }
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) { TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -146,7 +173,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
"Same number of input side packets and output streams is required.")); "Same number of input side packets and output streams is required."));
} }
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) { TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -248,7 +275,50 @@ TEST(SidePacketToStreamCalculator, AtTick) {
tick_and_verify(/*at_timestamp=*/1025); tick_and_verify(/*at_timestamp=*/1025);
} }
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) { TEST(SidePacketToStreamCalculator, AtFirstTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value))));
output_packets.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(
@ -302,6 +372,62 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
tick_and_verify(/*at_timestamp=*/1025); tick_and_verify(/*at_timestamp=*/1025);
} }
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_FIRST_TICK:0:packet0"
output_stream: "AT_FIRST_TICK:1:packet1"
}
)pb");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets0,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value0))));
EXPECT_THAT(output_packets1,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value1))));
output_packets0.clear();
output_packets1.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets0, IsEmpty());
EXPECT_THAT(output_packets1, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTimestamp) { TEST(SidePacketToStreamCalculator, AtTimestamp) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
@ -334,7 +460,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
EXPECT_EQ(expected_value, output_packets.back().Get<int>()); EXPECT_EQ(expected_value, output_packets.back().Get<int>());
} }
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) { TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>( ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( R"pb(

2
mediapipe/calculators/core/value_or_default_calculator_test.cc
View File

@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue)); ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
} }
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) { TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
// Check that when we provide the inputs not on time - we don't get them. // Check that when we provide the inputs not on time - we don't get them.
ValueOrDefaultRunner runner; ValueOrDefaultRunner runner;
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231}; const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};

2
mediapipe/calculators/image/affine_transformation_runner_opencv.cc
View File

@ -59,7 +59,7 @@ class OpenCvRunner
const ImageFrame& input, const std::array<float, 16>& matrix, const ImageFrame& input, const std::array<float, 16>& matrix,
const AffineTransformation::Size& size, const AffineTransformation::Size& size,
AffineTransformation::BorderMode border_mode) override { AffineTransformation::BorderMode border_mode) override {
// OpenCV warpAffine works in absolute coordinates, so the transfom (which // OpenCV warpAffine works in absolute coordinates, so the transform (which
// accepts and produces relative coordinates) should be adjusted to first // accepts and produces relative coordinates) should be adjusted to first
// normalize coordinates and then scale them. // normalize coordinates and then scale them.
// clang-format off // clang-format off

2
mediapipe/calculators/image/image_clone_calculator.cc
View File

@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
} }
#else #else
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true)); cc, /*request_gpu_as_optional=*/true));
#endif // MEDIAPIPE_DISABLE_GPU #endif // MEDIAPIPE_DISABLE_GPU
return absl::OkStatus(); return absl::OkStatus();
} }

2
mediapipe/calculators/image/image_cropping_calculator.proto
View File

@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
} }
// Output texture buffer dimensions. The values defined in the options will be // Output texture buffer dimensions. The values defined in the options will be
// overriden by the WIDTH and HEIGHT input streams if they exist. // overridden by the WIDTH and HEIGHT input streams if they exist.
optional int32 width = 1; optional int32 width = 1;
optional int32 height = 2; optional int32 height = 2;

6
mediapipe/calculators/image/image_file_properties_calculator.cc
View File

@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
return focal_length_pixels; return focal_length_pixels;
} }
absl::StatusOr<ImageFileProperties> GetImageFileProperites( absl::StatusOr<ImageFileProperties> GetImageFileProperties(
const std::string& image_bytes) { const std::string& image_bytes) {
easyexif::EXIFInfo result; easyexif::EXIFInfo result;
int code = result.parseFrom(image_bytes); int code = result.parseFrom(image_bytes);
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
if (cc->InputSidePackets().NumEntries() == 1) { if (cc->InputSidePackets().NumEntries() == 1) {
const std::string& image_bytes = const std::string& image_bytes =
cc->InputSidePackets().Index(0).Get<std::string>(); cc->InputSidePackets().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
read_properties_ = true; read_properties_ = true;
} }
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
return absl::OkStatus(); return absl::OkStatus();
} }
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>(); const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
read_properties_ = true; read_properties_ = true;
} }
if (read_properties_) { if (read_properties_) {

2
mediapipe/calculators/image/segmentation_smoothing_calculator.cc
View File

@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true)); cc, /*request_gpu_as_optional=*/true));
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
return absl::OkStatus(); return absl::OkStatus();

2
mediapipe/calculators/image/warp_affine_calculator.cc
View File

@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> || if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
std::is_same_v<InterfaceT, WarpAffineCalculator>) { std::is_same_v<InterfaceT, WarpAffineCalculator>) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true)); cc, /*request_gpu_as_optional=*/true));
} }
return absl::OkStatus(); return absl::OkStatus();
} }

2
mediapipe/calculators/image/warp_affine_calculator_test.cc
View File

@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
.IgnoreError(); .IgnoreError();
mediapipe::GetRotatedSubRectToRectTransformMatrix( mediapipe::GetRotatedSubRectToRectTransformMatrix(
roi_absolute, input.cols, input.rows, roi_absolute, input.cols, input.rows,
/*flip_horizontaly=*/false, &transform_mat); /*flip_horizontally=*/false, &transform_mat);
return transform_mat; return transform_mat;
} }

2
mediapipe/calculators/image/yuv_to_image_calculator.cc
View File

@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka // The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
// YV21) format (as per the `fourcc()` property). This covers the most commonly // YV21) format (as per the `fourcc()` property). This covers the most commonly
// used YUV image formats used on mobile devices. Other formats are not // used YUV image formats used on mobile devices. Other formats are not
// supported and wil result in an `InvalidArgumentError`. // supported and will result in an `InvalidArgumentError`.
class YUVToImageCalculator : public Node { class YUVToImageCalculator : public Node {
public: public:
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"}; static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};

131
mediapipe/calculators/tensor/BUILD
View File

@ -657,6 +657,7 @@ cc_library(
}), }),
deps = [ deps = [
":tensor_converter_calculator_cc_proto", ":tensor_converter_calculator_cc_proto",
":tensor_converter_cpu",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port", "//mediapipe/framework:port",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
@ -665,6 +666,7 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor", "//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util", "//mediapipe/util:resource_util",
@ -674,10 +676,17 @@ cc_library(
"@com_google_absl//absl/log:check", "@com_google_absl//absl/log:check",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"], "//conditions:default": [
"tensor_converter_calculator_gpu_deps",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}) + select({ }) + select({
"//mediapipe:apple": [ "//mediapipe:apple": [
"//third_party/apple_frameworks:MetalKit", "//third_party/apple_frameworks:MetalKit",
@ -687,6 +696,35 @@ cc_library(
alwayslink = 1, alwayslink = 1,
) )
cc_library(
name = "tensor_converter_cpu",
srcs = ["tensor_converter_cpu.cc"],
hdrs = ["tensor_converter_cpu.h"],
deps = [
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensor_converter_cpu_test",
srcs = ["tensor_converter_cpu_test.cc"],
deps = [
":tensor_converter_cpu",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"//mediapipe/util:image_test_utils",
],
)
cc_library( cc_library(
name = "tensor_converter_calculator_gpu_deps", name = "tensor_converter_calculator_gpu_deps",
visibility = ["//visibility:private"], visibility = ["//visibility:private"],
@ -1414,6 +1452,8 @@ cc_library(
}), }),
deps = [ deps = [
":tensors_to_segmentation_calculator_cc_proto", ":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port", "//mediapipe/framework:port",
@ -1421,9 +1461,11 @@ cc_library(
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor", "//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util", "//mediapipe/util:resource_util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span", "@com_google_absl//absl/types:span",
@ -1434,6 +1476,7 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:shader_util", "//mediapipe/gpu:shader_util",
], ],
}) + selects.with_or({ }) + selects.with_or({
@ -1453,19 +1496,96 @@ cc_library(
}) + select({ }) + select({
"//mediapipe/framework/port:disable_opencv": [], "//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [ "//conditions:default": [
"//mediapipe/framework/formats:image_opencv", ":tensors_to_segmentation_converter_opencv",
"//mediapipe/framework/port:opencv_imgproc",
], ],
}), }),
alwayslink = 1, alwayslink = 1,
) )
cc_library(
name = "tensors_to_segmentation_utils",
srcs = ["tensors_to_segmentation_utils.cc"],
hdrs = ["tensors_to_segmentation_utils.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensors_to_segmentation_utils_test",
srcs = ["tensors_to_segmentation_utils_test.cc"],
deps = [
":tensors_to_segmentation_utils",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter",
hdrs = ["tensors_to_segmentation_converter.h"],
deps = [
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter_opencv",
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_calculator_test_utils",
testonly = 1,
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "tensors_to_segmentation_calculator_test_utils_test",
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework/port:gtest_main",
],
)
cc_test( cc_test(
name = "tensors_to_segmentation_calculator_test", name = "tensors_to_segmentation_calculator_test",
srcs = ["tensors_to_segmentation_calculator_test.cc"], srcs = ["tensors_to_segmentation_calculator_test.cc"],
deps = [ deps = [
":tensors_to_segmentation_calculator", ":tensors_to_segmentation_calculator",
":tensors_to_segmentation_calculator_cc_proto", ":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet", "//mediapipe/framework:packet",
@ -1476,11 +1596,6 @@ cc_test(
"//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
], ],
) )

4
mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
View File

@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
// Non-streaming mode: when "stream_mode" is set to false in the calculator // Non-streaming mode: when "stream_mode" is set to false in the calculator
// options, the calculators treats the packets in the input audio stream as // options, the calculators treats the packets in the input audio stream as
// a batch of unrelated audio buffers. In each Process() call, the input // a batch of unrelated audio buffers. In each Process() call, the input
// buffer will be frist resampled, and framed as fixed-sized, possibly // buffer will be first resampled, and framed as fixed-sized, possibly
// overlapping tensors. The last tensor produced by a Process() invocation // overlapping tensors. The last tensor produced by a Process() invocation
// will be zero-padding if the remaining samples are insufficient. As the // will be zero-padding if the remaining samples are insufficient. As the
// calculator treats the input packets as unrelated, all samples will be // calculator treats the input packets as unrelated, all samples will be
@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
public: public:
static constexpr Input<Matrix> kAudioIn{"AUDIO"}; static constexpr Input<Matrix> kAudioIn{"AUDIO"};
// TODO: Removes this optional input stream when the "AUDIO" stream // TODO: Removes this optional input stream when the "AUDIO" stream
// uses the new mediapipe audio data containers that carry audio metatdata, // uses the new mediapipe audio data containers that carry audio metadata,
// such as sample rate. // such as sample rate.
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"}; static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"}; static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};

2
mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
View File

@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
// will be converted into tensors. // will be converted into tensors.
optional double target_sample_rate = 4; optional double target_sample_rate = 4;
// Whether to treat the input audio stream as a continous stream or a batch // Whether to treat the input audio stream as a continuous stream or a batch
// of unrelated audio buffers. // of unrelated audio buffers.
optional bool stream_mode = 5 [default = true]; optional bool stream_mode = 5 [default = true];

4
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
View File

@ -82,7 +82,7 @@ namespace api2 {
// //
// Outputs: // Outputs:
// TENSORS - std::vector<Tensor> // TENSORS - std::vector<Tensor>
// Vector containing a single Tensor populated with an extrated RGB image. // Vector containing a single Tensor populated with an extracted RGB image.
// MATRIX - std::array<float, 16> @Optional // MATRIX - std::array<float, 16> @Optional
// An std::array<float, 16> representing a 4x4 row-major-order matrix that // An std::array<float, 16> representing a 4x4 row-major-order matrix that
// maps a point on the input image to a point on the output tensor, and // maps a point on the input image to a point on the output tensor, and
@ -212,7 +212,7 @@ class ImageToTensorCalculator : public Node {
std::array<float, 16> matrix; std::array<float, 16> matrix;
GetRotatedSubRectToRectTransformMatrix( GetRotatedSubRectToRectTransformMatrix(
roi, image->width(), image->height(), roi, image->width(), image->height(),
/*flip_horizontaly=*/false, &matrix); /*flip_horizontally=*/false, &matrix);
kOutMatrix(cc).Send(std::move(matrix)); kOutMatrix(cc).Send(std::move(matrix));
} }

2
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
View File

@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) { } else if (image_channels == 1) {
return ImageFormat::GRAY8; return ImageFormat::GRAY8;
} }
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels; ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
} }
Packet MakeImageFramePacket(cv::Mat input) { Packet MakeImageFramePacket(cv::Mat input) {

8
mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc
View File

@ -57,7 +57,7 @@ class SubRectExtractorGl {
absl::Status ExtractSubRectToBuffer( absl::Status ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture, const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect, const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta, bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size, const tflite::gpu::HW& destination_size,
tflite::gpu::gl::CommandQueue* command_queue, tflite::gpu::gl::CommandQueue* command_queue,
tflite::gpu::gl::GlBuffer* destination); tflite::gpu::gl::GlBuffer* destination);
@ -154,13 +154,13 @@ void main() {
absl::Status SubRectExtractorGl::ExtractSubRectToBuffer( absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture, const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect, const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
bool flip_horizontaly, float alpha, float beta, bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size, const tflite::gpu::HW& destination_size,
tflite::gpu::gl::CommandQueue* command_queue, tflite::gpu::gl::CommandQueue* command_queue,
tflite::gpu::gl::GlBuffer* destination) { tflite::gpu::gl::GlBuffer* destination) {
std::array<float, 16> transform_mat; std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w, GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
texture_size.h, flip_horizontaly, texture_size.h, flip_horizontally,
&transform_mat); &transform_mat);
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0)); MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter {
input_texture, input_texture,
tflite::gpu::HW(source_texture.height(), source_texture.width()), tflite::gpu::HW(source_texture.height(), source_texture.width()),
roi, roi,
/*flip_horizontaly=*/false, transform.scale, transform.offset, /*flip_horizontally=*/false, transform.scale, transform.offset,
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]), tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
command_queue_.get(), &output)); command_queue_.get(), &output));

10
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc
View File

@ -199,7 +199,7 @@ class GlProcessor : public ImageToTensorConverter {
range_min, range_max)); range_min, range_max));
auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView(); auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi, MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
/*flip_horizontaly=*/false, /*flip_horizontally=*/false,
transform.scale, transform.offset, transform.scale, transform.offset,
output_shape, &tensor_view)); output_shape, &tensor_view));
return absl::OkStatus(); return absl::OkStatus();
@ -210,7 +210,7 @@ class GlProcessor : public ImageToTensorConverter {
absl::Status ExtractSubRect(const mediapipe::GlTexture& texture, absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
const RotatedRect& sub_rect, const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta, bool flip_horizontally, float alpha, float beta,
const Tensor::Shape& output_shape, const Tensor::Shape& output_shape,
Tensor::OpenGlTexture2dView* output) { Tensor::OpenGlTexture2dView* output) {
const int output_height = output_shape.dims[1]; const int output_height = output_shape.dims[1];
@ -263,13 +263,13 @@ class GlProcessor : public ImageToTensorConverter {
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) { if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
GetTransposedRotatedSubRectToRectTransformMatrix( GetTransposedRotatedSubRectToRectTransformMatrix(
sub_rect, texture.width(), texture.height(), flip_horizontaly, sub_rect, texture.width(), texture.height(), flip_horizontally,
&transform_mat); &transform_mat);
glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data()); glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
} else { } else {
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(), GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
texture.height(), flip_horizontaly, texture.height(),
&transform_mat); flip_horizontally, &transform_mat);
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data()); glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
} }

10
mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc
View File

@ -179,13 +179,13 @@ class SubRectExtractorMetal {
} }
absl::Status Execute(id<MTLTexture> input_texture, absl::Status Execute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect, bool flip_horizontaly, const RotatedRect& sub_rect, bool flip_horizontally,
float alpha, float beta, float alpha, float beta,
const tflite::gpu::HW& destination_size, const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer, id<MTLCommandBuffer> command_buffer,
id<MTLBuffer> destination) { id<MTLBuffer> destination) {
auto output_texture = MTLTextureWithBuffer(destination_size, destination); auto output_texture = MTLTextureWithBuffer(destination_size, destination);
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha, return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
beta, destination_size, command_buffer, beta, destination_size, command_buffer,
output_texture); output_texture);
} }
@ -211,7 +211,7 @@ class SubRectExtractorMetal {
absl::Status InternalExecute(id<MTLTexture> input_texture, absl::Status InternalExecute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect, const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta, bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size, const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer, id<MTLCommandBuffer> command_buffer,
id<MTLTexture> output_texture) { id<MTLTexture> output_texture) {
@ -223,7 +223,7 @@ class SubRectExtractorMetal {
std::array<float, 16> transform_mat; std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width, GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
input_texture.height, input_texture.height,
flip_horizontaly, &transform_mat); flip_horizontally, &transform_mat);
id<MTLBuffer> transform_mat_buffer = id<MTLBuffer> transform_mat_buffer =
[device_ newBufferWithBytes:&transform_mat [device_ newBufferWithBytes:&transform_mat
length:sizeof(transform_mat) length:sizeof(transform_mat)
@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
MtlBufferView::GetWriteView(output_tensor, command_buffer); MtlBufferView::GetWriteView(output_tensor, command_buffer);
MP_RETURN_IF_ERROR(extractor_->Execute( MP_RETURN_IF_ERROR(extractor_->Execute(
texture, roi, texture, roi,
/*flip_horizontaly=*/false, transform.scale, transform.offset, /*flip_horizontally=*/false, transform.scale, transform.offset,
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]), tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
command_buffer, buffer_view.buffer())); command_buffer, buffer_view.buffer()));
[command_buffer commit]; [command_buffer commit];

8
mediapipe/calculators/tensor/image_to_tensor_utils.cc
View File

@ -92,7 +92,7 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
int rect_width, int rect_height, int rect_width, int rect_height,
bool flip_horizontaly, bool flip_horizontally,
std::array<float, 16>* matrix_ptr) { std::array<float, 16>* matrix_ptr) {
std::array<float, 16>& matrix = *matrix_ptr; std::array<float, 16>& matrix = *matrix_ptr;
// The resulting matrix is multiplication of below commented out matrices: // The resulting matrix is multiplication of below commented out matrices:
@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
// {0.0f, 0.0f, a, 0.0f} // {0.0f, 0.0f, a, 0.0f}
// {0.0f, 0.0f, 0.0f, 1.0f} // {0.0f, 0.0f, 0.0f, 1.0f}
const float flip = flip_horizontaly ? -1 : 1; const float flip = flip_horizontally ? -1 : 1;
// Matrix for optional horizontal flip around middle of output image. // Matrix for optional horizontal flip around middle of output image.
// { fl , 0.0f, 0.0f, 0.0f} // { fl , 0.0f, 0.0f, 0.0f}
// { 0.0f, 1.0f, 0.0f, 0.0f} // { 0.0f, 1.0f, 0.0f, 0.0f}
@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
void GetTransposedRotatedSubRectToRectTransformMatrix( void GetTransposedRotatedSubRectToRectTransformMatrix(
const RotatedRect& sub_rect, int rect_width, int rect_height, const RotatedRect& sub_rect, int rect_width, int rect_height,
bool flip_horizontaly, std::array<float, 16>* matrix_ptr) { bool flip_horizontally, std::array<float, 16>* matrix_ptr) {
std::array<float, 16>& matrix = *matrix_ptr; std::array<float, 16>& matrix = *matrix_ptr;
// See comments in GetRotatedSubRectToRectTransformMatrix for detailed // See comments in GetRotatedSubRectToRectTransformMatrix for detailed
// calculations. // calculations.
const float a = sub_rect.width; const float a = sub_rect.width;
const float b = sub_rect.height; const float b = sub_rect.height;
const float flip = flip_horizontaly ? -1 : 1; const float flip = flip_horizontally ? -1 : 1;
const float c = std::cos(sub_rect.rotation); const float c = std::cos(sub_rect.rotation);
const float d = std::sin(sub_rect.rotation); const float d = std::sin(sub_rect.rotation);
const float e = sub_rect.center_x; const float e = sub_rect.center_x;

10
mediapipe/calculators/tensor/image_to_tensor_utils.h
View File

@ -74,7 +74,7 @@ absl::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
// Represents a transformation of value which involves scaling and offsetting. // Represents a transformation of value which involves scaling and offsetting.
// To apply transformation: // To apply transformation:
// ValueTransformation transform = ... // ValueTransformation transform = ...
// float transformed_value = transform.scale * value + transfrom.offset; // float transformed_value = transform.scale * value + transform.offset;
struct ValueTransformation { struct ValueTransformation {
float scale; float scale;
float offset; float offset;
@ -99,11 +99,11 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
// @sub_rect - rotated sub rect in absolute coordinates // @sub_rect - rotated sub rect in absolute coordinates
// @rect_width - rect width // @rect_width - rect width
// @rect_height - rect height // @rect_height - rect height
// @flip_horizontaly - we need to flip the output buffer. // @flip_horizontally - we need to flip the output buffer.
// @matrix - 4x4 matrix (array of 16 elements) to populate // @matrix - 4x4 matrix (array of 16 elements) to populate
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
int rect_width, int rect_height, int rect_width, int rect_height,
bool flip_horizontaly, bool flip_horizontally,
std::array<float, 16>* matrix); std::array<float, 16>* matrix);
// Returns the transpose of the matrix found with // Returns the transpose of the matrix found with
@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
// @sub_rect - rotated sub rect in absolute coordinates // @sub_rect - rotated sub rect in absolute coordinates
// @rect_width - rect width // @rect_width - rect width
// @rect_height - rect height // @rect_height - rect height
// @flip_horizontaly - we need to flip the output buffer. // @flip_horizontally - we need to flip the output buffer.
// @matrix - 4x4 matrix (array of 16 elements) to populate // @matrix - 4x4 matrix (array of 16 elements) to populate
void GetTransposedRotatedSubRectToRectTransformMatrix( void GetTransposedRotatedSubRectToRectTransformMatrix(
const RotatedRect& sub_rect, int rect_width, int rect_height, const RotatedRect& sub_rect, int rect_width, int rect_height,
bool flip_horizontaly, std::array<float, 16>* matrix); bool flip_horizontally, std::array<float, 16>* matrix);
// Validates the output dimensions set in the option proto. The input option // Validates the output dimensions set in the option proto. The input option
// proto is expected to have to following fields: // proto is expected to have to following fields:

125
mediapipe/calculators/tensor/tensor_converter_calculator.cc
View File

@ -14,6 +14,7 @@
#include <cstdint> #include <cstdint>
#include <string> #include <string>
#include <utility>
#include <vector> #include <vector>
#include "absl/log/absl_check.h" #include "absl/log/absl_check.h"
@ -21,17 +22,22 @@
#include "absl/status/status.h" #include "absl/status/status.h"
#include "absl/status/statusor.h" #include "absl/status/statusor.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
#include "absl/types/optional.h"
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h" #include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_origin.pb.h" #include "mediapipe/gpu/gpu_origin.pb.h"
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/gpu_buffer.h"
#if MEDIAPIPE_METAL_ENABLED #if MEDIAPIPE_METAL_ENABLED
#import <CoreVideo/CoreVideo.h> #import <CoreVideo/CoreVideo.h>
@ -94,16 +100,13 @@ absl::StatusOr<bool> ShouldFlipVertically(
} }
} }
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kTensorsTag[] = "TENSORS"; constexpr char kTensorsTag[] = "TENSORS";
constexpr char kMatrixTag[] = "MATRIX"; constexpr char kMatrixTag[] = "MATRIX";
constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
} // namespace } // namespace
namespace mediapipe { namespace mediapipe {
@ -156,10 +159,6 @@ class TensorConverterCalculator : public CalculatorBase {
private: private:
absl::Status InitGpu(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc);
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu); absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame,
bool flip_vertically, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
absl::Status ProcessCPU(CalculatorContext* cc); absl::Status ProcessCPU(CalculatorContext* cc);
absl::Status ProcessGPU(CalculatorContext* cc); absl::Status ProcessGPU(CalculatorContext* cc);
@ -279,46 +278,21 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
} }
const auto& image_frame = const auto& image_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>(); cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
const int height = image_frame.Height(); MP_ASSIGN_OR_RETURN(Tensor output,
const int width = image_frame.Width(); ConvertImageFrameToTensorOnCpu(
const int channels = image_frame.NumberOfChannels(); image_frame,
const int channels_preserved = std::min(channels, max_num_channels_); output_range_.has_value() ? output_range_.value()
const mediapipe::ImageFormat::Format format = image_frame.Format(); : kDefaultOutputRange,
flip_vertically_, max_num_channels_));
if (!(format == mediapipe::ImageFormat::SRGBA || output_tensors->emplace_back(std::move(output));
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
output_tensors->emplace_back(
Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensors->back().GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
} else if (cc->Inputs().HasTag(kMatrixTag)) { } else if (cc->Inputs().HasTag(kMatrixTag)) {
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
return absl::OkStatus(); return absl::OkStatus();
} }
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>(); const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
const int height = matrix.rows(); MP_ASSIGN_OR_RETURN(Tensor output,
const int width = matrix.cols(); ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
const int channels = 1; output_tensors->emplace_back(std::move(output));
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
} else { } else {
return absl::OkStatus(); return absl::OkStatus();
} }
@ -669,67 +643,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
return absl::OkStatus(); return absl::OkStatus();
} }
template <class T>
absl::Status TensorConverterCalculator::NormalizeImage(
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels_);
const int channels_ignored = channels - channels_preserved;
if (output_range_.has_value()) {
// If the output float range is set and we are not using custom
// normalization, normalize the pixel values from [0, 255] to the specified
// output range.
RET_CHECK_NE(output_range_->first, output_range_->second);
const float scale = (output_range_->second - output_range_->first) / 255.0f;
const float bias = output_range_->first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
} else {
// [0,1], scale only (bias == 0)
// Verified that there are no precision issues with 1.0f / 255.0f expression
const float scale = 1.0f / 255.0f;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale;
}
image_ptr += channels_ignored;
}
}
}
return absl::OkStatus();
}
absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
float* tensor_ptr) {
if (row_major_matrix_) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
} // namespace mediapipe } // namespace mediapipe

2
mediapipe/calculators/tensor/tensor_converter_calculator.proto
View File

@ -32,7 +32,7 @@ message TensorConverterCalculatorOptions {
// Custom settings to override the internal scaling factors `div` and `sub`. // Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on // Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom // CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overriden, and // values take effect, the |zero_center| setting above will be overridden, and
// the normalized_value will be calculated as: // the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub. // normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false]; optional bool use_custom_normalization = 6 [default = false];

55
mediapipe/calculators/tensor/tensor_converter_calculator_test.cc
View File

@ -321,6 +321,61 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
} }
} }
TEST_F(TensorConverterCalculatorTest,
ShouldConvertImageWithDefaultOutputRange) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
zero_center: false
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
mat.at<uint8_t>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1);
// Get and process results.
const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1);
const Tensor* tensor = &tensor_vec[0];
// Calculate the expected normalized value:
float expected_value = 200.0 / 255.0;
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
auto view = tensor->GetCpuReadView();
float actual_value = *view.buffer<float>();
EXPECT_FLOAT_EQ(actual_value, expected_value);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(TensorConverterCalculatorTest, FlipVertically) { TEST_F(TensorConverterCalculatorTest, FlipVertically) {
CalculatorGraph graph; CalculatorGraph graph;
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =

145
mediapipe/calculators/tensor/tensor_converter_cpu.cc Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <algorithm>
#include <cstdint>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const int channels_ignored = channels - channels_preserved;
RET_CHECK_NE(output_range.first, output_range.second);
const float scale = (output_range.second - output_range.first) / 255.0f;
const float bias = output_range.first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
return absl::OkStatus();
}
} // namespace
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<float>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr) {
if (is_row_major_matrix) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const mediapipe::ImageFormat::Format format = image_frame.Format();
if (!(format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensor.GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
return output_tensor;
}
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix) {
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(
CopyMatrixToTensor(matrix, row_major_matrix,
output_tensor.GetCpuWriteView().buffer<float>()));
return output_tensor;
}
} // namespace mediapipe

61
mediapipe/calculators/tensor/tensor_converter_cpu.h Normal file
View File

@ -0,0 +1,61 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
// Converts an ImageFrame to a vector of Tensors.
// @flip_vertically enables to flip the image during conversion.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels);
// Converts a Matrix to a vector of Tensors.
// @row_major_matrix defines the ordering in the input matrix.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix);
// For testing only below.
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_

175
mediapipe/calculators/tensor/tensor_converter_cpu_test.cc Normal file
View File

@ -0,0 +1,175 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <cstdint>
#include <utility>
#include <vector>
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/util/image_test_utils.h"
namespace mediapipe {
namespace {
Matrix CreateTestMatrix(int num_rows, int num_columns) {
Matrix matrix(num_rows, num_columns);
for (int r = 0; r < num_rows; ++r) {
for (int c = 0; c < num_columns; ++c) {
matrix(r, c) = r * num_columns + c;
}
}
return matrix;
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i / test_matrix.cols();
const int column = i % test_matrix.cols();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i % test_matrix.rows();
const int column = i / test_matrix.rows();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(
tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
const auto range = std::make_pair(2.0f, 3.0f);
MP_EXPECT_OK(
NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
/*num_tensor_channels=*/1, tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
255.0f * (range.second - range.first) +
range.first);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int x = i % grey8_image_frame.Width();
const int y = i / grey8_image_frame.Width();
const int flipped_y = grey8_image_frame.Height() - y - 1;
const int index = flipped_y * grey8_image_frame.Width() + x;
EXPECT_FLOAT_EQ(
tensor_data[index],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
auto float_image_frame =
CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
float_image_frame.Width() * float_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
float_image_frame.PixelData())[i] /
255.0f);
}
}
TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
grey8_image_frame, {0.0f, 1.0f},
/*flip_vertically=*/false,
/*max_num_channels=*/1));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
++i) {
EXPECT_FLOAT_EQ(
tensor_ptr[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
}
}
TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
MP_ASSERT_OK_AND_ASSIGN(
Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
/*row_major_matrix=*/false));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < test_matrix.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
}
}
} // namespace
} // namespace mediapipe

2
mediapipe/calculators/tensor/tensors_to_classification_calculator.proto
View File

@ -34,7 +34,7 @@ message TensorsToClassificationCalculatorOptions {
repeated Entry entries = 1; repeated Entry entries = 1;
} }
// Score threshold for perserving the class. // Score threshold for preserving the class.
optional float min_score_threshold = 1; optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then // Number of highest scoring labels to output. If top_k is not positive then
// all labels are used. // all labels are used.

60
mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
View File

@ -15,7 +15,6 @@
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/types/span.h" #include "absl/types/span.h"
#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
@ -147,7 +146,7 @@ BoxFormat GetBoxFormat(const TensorsToDetectionsCalculatorOptions& options) {
// TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have // TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have
// 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints. // 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints.
// The size of the values must be (num_boxes * num_predicted_values). // The size of the values must be (num_boxes * num_predicted_values).
// Second tensor is the score tensor. The size of the valuse must be // Second tensor is the score tensor. The size of the values must be
// (num_boxes * num_classes). It's optional to pass in a third tensor // (num_boxes * num_classes). It's optional to pass in a third tensor
// for anchors (e.g. for SSD models) depend on the outputs of the // for anchors (e.g. for SSD models) depend on the outputs of the
// detection model. The size of anchor tensor must be (num_boxes * // detection model. The size of anchor tensor must be (num_boxes *
@ -215,7 +214,8 @@ class TensorsToDetectionsCalculator : public Node {
const int* detection_classes, const int* detection_classes,
std::vector<Detection>* output_detections); std::vector<Detection>* output_detections);
Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
float box_xmax, float score, int class_id, float box_xmax, absl::Span<const float> scores,
absl::Span<const int> class_ids,
bool flip_vertically); bool flip_vertically);
bool IsClassIndexAllowed(int class_index); bool IsClassIndexAllowed(int class_index);
@ -223,6 +223,7 @@ class TensorsToDetectionsCalculator : public Node {
int num_boxes_ = 0; int num_boxes_ = 0;
int num_coords_ = 0; int num_coords_ = 0;
int max_results_ = -1; int max_results_ = -1;
int classes_per_detection_ = 1;
BoxFormat box_output_format_ = BoxFormat box_output_format_ =
mediapipe::TensorsToDetectionsCalculatorOptions::YXHW; mediapipe::TensorsToDetectionsCalculatorOptions::YXHW;
@ -267,7 +268,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract(
if (CanUseGpu()) { if (CanUseGpu()) {
#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true)); cc, /*request_gpu_as_optional=*/true));
#elif MEDIAPIPE_METAL_ENABLED #elif MEDIAPIPE_METAL_ENABLED
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@ -484,6 +485,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
auto num_boxes_view = num_boxes_tensor->GetCpuReadView(); auto num_boxes_view = num_boxes_tensor->GetCpuReadView();
auto num_boxes = num_boxes_view.buffer<float>(); auto num_boxes = num_boxes_view.buffer<float>();
num_boxes_ = num_boxes[0]; num_boxes_ = num_boxes[0];
// The detection model with Detection_PostProcess op may output duplicate
// boxes with different classes, in the following format:
// num_boxes_tensor = [num_boxes]
// detection_classes_tensor = [box_1_class_1, box_1_class_2, ...]
// detection_scores_tensor = [box_1_score_1, box_1_score_2, ... ]
// detection_boxes_tensor = [box_1, box1, ... ]
// Each box repeats classes_per_detection_ times.
// Note Detection_PostProcess op is only supported in CPU.
RET_CHECK_EQ(max_detections % num_boxes_, 0);
classes_per_detection_ = max_detections / num_boxes_;
auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView(); auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView();
auto detection_boxes = detection_boxes_view.buffer<float>(); auto detection_boxes = detection_boxes_view.buffer<float>();
@ -493,8 +504,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
auto detection_classes_view = detection_classes_tensor->GetCpuReadView(); auto detection_classes_view = detection_classes_tensor->GetCpuReadView();
auto detection_classes_ptr = detection_classes_view.buffer<float>(); auto detection_classes_ptr = detection_classes_view.buffer<float>();
std::vector<int> detection_classes(num_boxes_); std::vector<int> detection_classes(num_boxes_ * classes_per_detection_);
for (int i = 0; i < num_boxes_; ++i) { for (int i = 0; i < detection_classes.size(); ++i) {
detection_classes[i] = static_cast<int>(detection_classes_ptr[i]); detection_classes[i] = static_cast<int>(detection_classes_ptr[i]);
} }
MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores, MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores,
@ -863,24 +874,25 @@ absl::Status TensorsToDetectionsCalculator::DecodeBoxes(
absl::Status TensorsToDetectionsCalculator::ConvertToDetections( absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
const float* detection_boxes, const float* detection_scores, const float* detection_boxes, const float* detection_scores,
const int* detection_classes, std::vector<Detection>* output_detections) { const int* detection_classes, std::vector<Detection>* output_detections) {
for (int i = 0; i < num_boxes_; ++i) { for (int i = 0; i < num_boxes_ * classes_per_detection_;
i += classes_per_detection_) {
if (max_results_ > 0 && output_detections->size() == max_results_) { if (max_results_ > 0 && output_detections->size() == max_results_) {
break; break;
} }
if (options_.has_min_score_thresh() &&
detection_scores[i] < options_.min_score_thresh()) {
continue;
}
if (!IsClassIndexAllowed(detection_classes[i])) {
continue;
}
const int box_offset = i * num_coords_; const int box_offset = i * num_coords_;
Detection detection = ConvertToDetection( Detection detection = ConvertToDetection(
/*box_ymin=*/detection_boxes[box_offset + box_indices_[0]], /*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
/*box_xmin=*/detection_boxes[box_offset + box_indices_[1]], /*box_xmin=*/detection_boxes[box_offset + box_indices_[1]],
/*box_ymax=*/detection_boxes[box_offset + box_indices_[2]], /*box_ymax=*/detection_boxes[box_offset + box_indices_[2]],
/*box_xmax=*/detection_boxes[box_offset + box_indices_[3]], /*box_xmax=*/detection_boxes[box_offset + box_indices_[3]],
detection_scores[i], detection_classes[i], options_.flip_vertically()); absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
options_.flip_vertically());
// if all the scores and classes are filtered out, we skip the empty
// detection.
if (detection.score().empty()) {
continue;
}
const auto& bbox = detection.location_data().relative_bounding_box(); const auto& bbox = detection.location_data().relative_bounding_box();
if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) || if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
std::isnan(bbox.height())) { std::isnan(bbox.height())) {
@ -910,11 +922,21 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
} }
Detection TensorsToDetectionsCalculator::ConvertToDetection( Detection TensorsToDetectionsCalculator::ConvertToDetection(
float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, float box_ymin, float box_xmin, float box_ymax, float box_xmax,
int class_id, bool flip_vertically) { absl::Span<const float> scores, absl::Span<const int> class_ids,
bool flip_vertically) {
Detection detection; Detection detection;
detection.add_score(score); for (int i = 0; i < scores.size(); ++i) {
detection.add_label_id(class_id); if (!IsClassIndexAllowed(class_ids[i])) {
continue;
}
if (options_.has_min_score_thresh() &&
scores[i] < options_.min_score_thresh()) {
continue;
}
detection.add_score(scores[i]);
detection.add_label_id(class_ids[i]);
}
LocationData* location_data = detection.mutable_location_data(); LocationData* location_data = detection.mutable_location_data();
location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);

2
mediapipe/calculators/tensor/tensors_to_detections_calculator.proto
View File

@ -75,7 +75,7 @@ message TensorsToDetectionsCalculatorOptions {
// representation has a bottom-left origin (e.g., in OpenGL). // representation has a bottom-left origin (e.g., in OpenGL).
optional bool flip_vertically = 18 [default = false]; optional bool flip_vertically = 18 [default = false];
// Score threshold for perserving decoded detections. // Score threshold for preserving decoded detections.
optional float min_score_thresh = 19; optional float min_score_thresh = 19;
// The maximum number of the detection results to return. If < 0, all // The maximum number of the detection results to return. If < 0, all

2
mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
View File

@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
kFlipVertically(cc).IsConnected())) { kFlipVertically(cc).IsConnected())) {
RET_CHECK(options_.has_input_image_height() && RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width()) options_.has_input_image_width())
<< "Must provide input width/height for using flipping when outputing " << "Must provide input width/height for using flipping when outputting "
"landmarks in absolute coordinates."; "landmarks in absolute coordinates.";
} }
return absl::OkStatus(); return absl::OkStatus();

223
mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
View File

@ -12,32 +12,35 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector> #include <vector>
#include "absl/strings/str_format.h" #include "absl/status/status.h"
#include "absl/types/span.h" #include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h" #include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/statusor.h" #include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/gpu_origin.pb.h" #include "mediapipe/gpu/gpu_origin.pb.h"
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h" #include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/shader_util.h" #include "mediapipe/gpu/shader_util.h"
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_OPENCV #if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/framework/formats/image_opencv.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#endif // !MEDIAPIPE_DISABLE_OPENCV #endif // !MEDIAPIPE_DISABLE_OPENCV
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
@ -62,37 +65,9 @@ namespace {
constexpr int kWorkgroupSize = 8; // Block size for GPU shader. constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size) { // NOLINT
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
constexpr char kTensorsTag[] = "TENSORS"; constexpr char kTensorsTag[] = "TENSORS";
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
constexpr char kMaskTag[] = "MASK"; constexpr char kMaskTag[] = "MASK";
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace } // namespace
namespace mediapipe { namespace mediapipe {
@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
private: private:
absl::Status LoadOptions(CalculatorContext* cc); absl::Status LoadOptions(CalculatorContext* cc);
absl::Status InitGpu(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc);
absl::Status ProcessGpu(CalculatorContext* cc); absl::Status ProcessGpu(CalculatorContext* cc,
absl::Status ProcessCpu(CalculatorContext* cc); const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width,
int output_height);
void GlRender(); void GlRender();
bool DoesGpuTextureStartAtBottom() { bool DoesGpuTextureStartAtBottom() {
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
} }
absl::Status InitConverterIfNecessary() {
#if !MEDIAPIPE_DISABLE_OPENCV #if !MEDIAPIPE_DISABLE_OPENCV
template <class T> if (!cpu_converter_) {
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
}
#else
RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV #endif // !MEDIAPIPE_DISABLE_OPENCV
::mediapipe::TensorsToSegmentationCalculatorOptions options_; return absl::OkStatus();
}
mediapipe::TensorsToSegmentationCalculatorOptions options_;
std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
mediapipe::GlCalculatorHelper gpu_helper_; mediapipe::GlCalculatorHelper gpu_helper_;
@ -208,7 +192,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract(
if (CanUseGpu()) { if (CanUseGpu()) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true)); cc, /*request_gpu_as_optional=*/true));
#if MEDIAPIPE_METAL_ENABLED #if MEDIAPIPE_METAL_ENABLED
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif // MEDIAPIPE_METAL_ENABLED #endif // MEDIAPIPE_METAL_ENABLED
@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
MP_ASSIGN_OR_RETURN(auto hwc, MP_ASSIGN_OR_RETURN(auto hwc,
GetHwcFromDims(input_tensors[0].shape().dims)); GetHwcFromDims(input_tensors[0].shape().dims));
int tensor_channels = std::get<2>(hwc); int tensor_channels = std::get<2>(hwc);
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
switch (options_.activation()) { switch (options_.activation()) {
case Options::NONE: case Options::NONE:
RET_CHECK_EQ(tensor_channels, 1); RET_CHECK_EQ(tensor_channels, 1);
@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
} }
} }
// Get dimensions.
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
if (use_gpu) { if (use_gpu) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
if (!gpu_initialized_) { if (!gpu_initialized_) {
@ -286,8 +281,11 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { MP_RETURN_IF_ERROR(
MP_RETURN_IF_ERROR(ProcessGpu(cc)); gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
output_height, hwc]() -> absl::Status {
MP_RETURN_IF_ERROR(
ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
return absl::OkStatus(); return absl::OkStatus();
})); }));
#else #else
@ -295,7 +293,13 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
} else { } else {
#if !MEDIAPIPE_DISABLE_OPENCV #if !MEDIAPIPE_DISABLE_OPENCV
MP_RETURN_IF_ERROR(ProcessCpu(cc)); // Lazily initialize converter.
MP_RETURN_IF_ERROR(InitConverterIfNecessary());
MP_ASSIGN_OR_RETURN(
std::unique_ptr<Image> output_mask,
cpu_converter_->Convert(input_tensors, output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
cc->InputTimestamp());
#else #else
RET_CHECK_FAIL() << "OpenCV processing disabled."; RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV #endif // !MEDIAPIPE_DISABLE_OPENCV
@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status TensorsToSegmentationCalculator::ProcessCpu(
CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_OPENCV
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
#endif // !MEDIAPIPE_DISABLE_OPENCV
return absl::OkStatus();
}
#if !MEDIAPIPE_DISABLE_OPENCV
template <class T>
absl::Status TensorsToSegmentationCalculator::ApplyActivation(
cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
#endif // !MEDIAPIPE_DISABLE_OPENCV
// Steps: // Steps:
// 1. receive tensor // 1. receive tensor
// 2. process segmentation tensor into small mask // 2. process segmentation tensor into small mask
// 3. upsample small mask into output mask to be same size as input image // 3. upsample small mask into output mask to be same size as input image
absl::Status TensorsToSegmentationCalculator::ProcessGpu( absl::Status TensorsToSegmentationCalculator::ProcessGpu(
CalculatorContext* cc) { CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width, int output_height) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc; auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask texture. // Create initial working mask texture.
#if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() {
absl::Status TensorsToSegmentationCalculator::LoadOptions( absl::Status TensorsToSegmentationCalculator::LoadOptions(
CalculatorContext* cc) { CalculatorContext* cc) {
// Get calculator options specified in the graph. // Get calculator options specified in the graph.
options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
return absl::OkStatus(); return absl::OkStatus();
} }
@ -826,7 +713,7 @@ void main() {
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
// Shader defines. // Shader defines.
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
const std::string output_layer_index = const std::string output_layer_index =
"\n#define OUTPUT_LAYER_INDEX int(" + "\n#define OUTPUT_LAYER_INDEX int(" +
std::to_string(options_.output_layer_index()) + ")"; std::to_string(options_.output_layer_index()) + ")";

191
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc
View File

@ -17,10 +17,8 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "absl/log/log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image.h"
@ -30,7 +28,6 @@
#include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/timestamp.h"
@ -40,62 +37,17 @@ namespace {
using ::testing::SizeIs; using ::testing::SizeIs;
using ::testing::TestWithParam; using ::testing::TestWithParam;
using Options = mediapipe::TensorsToSegmentationCalculatorOptions; using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
std::string ActivationTypeToString(Options::Activation activation) { using TensorsToSegmentationCalculatorTest =
switch (activation) { TestWithParam<test_utils::FormattingTestCase>;
case Options::NONE:
return "NONE";
case Options::SIGMOID:
return "SIGMOID";
case Options::SOFTMAX:
return "SOFTMAX";
default:
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
}
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
Options::Activation activation;
int rows;
int cols;
int channels;
};
using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
// Currently only useable for tests with no output resize.
TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
const FormattingTestCase& test_case = GetParam(); const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
std::vector<float> inputs = test_case.inputs; rows_new, cols_new, channels, max_abs_diff] = GetParam();
std::vector<float> expected_outputs = test_case.expected_outputs;
Options::Activation activation = test_case.activation;
int rows = test_case.rows;
int cols = test_case.cols;
int channels = test_case.channels;
std::string string_config = absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size"
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $0
}
}
}
)pb",
ActivationTypeToString(activation));
auto graph_config = auto graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(string_config); test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
std::vector<Packet> output_packets; std::vector<Packet> output_packets;
tool::AddVectorSink("image_as_mask", &graph_config, &output_packets); tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
@ -119,28 +71,34 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
MP_ASSERT_OK(graph.AddPacketToInputStream( MP_ASSERT_OK(graph.AddPacketToInputStream(
"tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0)))); "tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0))));
} }
// The output size is defined as pair(new_width, new_height).
MP_ASSERT_OK(graph.AddPacketToInputStream( MP_ASSERT_OK(graph.AddPacketToInputStream(
"size", "size", mediapipe::Adopt(new std::pair<int, int>(cols_new, rows_new))
mediapipe::Adopt(new std::pair<int, int>(rows, cols)).At(Timestamp(0)))); .At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle()); MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_packets, SizeIs(1)); ASSERT_THAT(output_packets, SizeIs(1));
const Image& image_as_mask = output_packets[0].Get<Image>(); const Image& image_as_mask = output_packets[0].Get<Image>();
EXPECT_FALSE(image_as_mask.UsesGpu());
std::shared_ptr<cv::Mat> result_mat = formats::MatView(&image_as_mask); std::shared_ptr<cv::Mat> result_mat = formats::MatView(&image_as_mask);
EXPECT_EQ(result_mat->rows, rows); EXPECT_EQ(result_mat->rows, rows_new);
EXPECT_EQ(result_mat->cols, cols); EXPECT_EQ(result_mat->cols, cols_new);
EXPECT_EQ(result_mat->channels(), channels); EXPECT_EQ(result_mat->channels(), 1);
// Compare the real result with the expected result. // Compare the real result with the expected result.
cv::Mat expected_result = cv::Mat( cv::Mat expected_result =
rows, cols, CV_32FC1, const_cast<float*>(expected_outputs.data())); cv::Mat(rows_new, cols_new, CV_32FC1,
const_cast<float*>(expected_outputs.data()));
cv::Mat diff; cv::Mat diff;
cv::absdiff(*result_mat, expected_result, diff); cv::absdiff(*result_mat, expected_result, diff);
double max_val; double max_val;
cv::minMaxLoc(diff, nullptr, &max_val); cv::minMaxLoc(diff, nullptr, &max_val);
// Expects the maximum absolute pixel-by-pixel difference is less than 1e-5.
// This delta is for passthorugh accuracy only. // The max allowable diff between output and expected output varies between
EXPECT_LE(max_val, 1e-5); // tests.
EXPECT_LE(max_val, max_abs_diff);
MP_ASSERT_OK(graph.CloseInputStream("tensors")); MP_ASSERT_OK(graph.CloseInputStream("tensors"));
MP_ASSERT_OK(graph.CloseInputStream("size")); MP_ASSERT_OK(graph.CloseInputStream("size"));
@ -149,18 +107,97 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P(
TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest, TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
testing::ValuesIn<FormattingTestCase>({ testing::ValuesIn<test_utils::FormattingTestCase>({
{/*test_name=*/"NoActivationAndNoOutputResize", {.test_name = "NoActivationAndNoOutputResize",
/*inputs=*/ .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 12.0, 13.0, 14.0, 15.0, 16.0},
14.0, 15.0, 16.0}, .expected_outputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
/*expected_outputs=*/ 11.0, 12.0, 13.0, 14.0, 15.0, 16.0},
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, .activation = Options::NONE,
14.0, 15.0, 16.0}, .rows = 4,
/*activation=*/Options::NONE, .cols = 4,
/*rows=*/4, .rows_new = 4,
/*cols=*/4, .cols_new = 4,
/*channels=*/1}, .channels = 1,
.max_abs_diff = 1e-7},
{.test_name = "OutputResizeOnly",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {1, 1.5, 2.166667, 2.833333, 3.5, 4,
3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8,
7, 7.5, 8.166667, 8.833333, 9.5, 10,
10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2,
13, 13.5, 14.166667, 14.833333, 15.5, 16},
.activation = Options::NONE,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SigmoidActivationWithNoOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {0.731059, 0.880797, 0.952574, 0.982014, 0.993307,
0.997527, 0.999089, 0.999665, 0.999877, 0.999955,
0.999983, 0.999994, 0.999998, 0.999999, 1.0, 1.0},
.activation = Options::SIGMOID,
.rows = 4,
.cols = 4,
.rows_new = 4,
.cols_new = 4,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SigmoidActivationWithOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {0.731059, 0.805928, 0.89276, 0.940611, 0.967294,
0.982014, 0.914633, 0.93857, 0.966279, 0.981363,
0.989752, 0.994369, 0.996592, 0.997666, 0.998873,
0.999404, 0.999683, 0.999829, 0.999913, 0.99994,
0.999971, 0.999985, 0.999992, 0.999996, 0.999998,
0.999998, 0.999999, 1.0, 1.0, 1.0},
.activation = Options::SIGMOID,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SoftmaxActivationWithNoOutputResize",
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
.expected_outputs = {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574,
0.000911051, 0.952574, 0.924142, 0.731059,
0.731059, 0.24974, 0.937027, 0.689974, 0.990048,
0.0060598, 0.28905},
.activation = Options::SOFTMAX,
.rows = 4,
.cols = 4,
.rows_new = 4,
.cols_new = 4,
.channels = 2,
.max_abs_diff = 1e-6},
{.test_name = "SoftmaxActivationWithOutputResize",
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
.expected_outputs = {0.731059, 0.425131, 0.246135, 0.753865, 0.445892,
0.0109869, 0.886119, 0.461259, 0.185506, 0.781934,
0.790618, 0.650195, 0.841816, 0.603901, 0.40518,
0.561962, 0.765871, 0.930584, 0.718733, 0.763744,
0.703402, 0.281989, 0.459635, 0.742634, 0.689974,
0.840011, 0.82605, 0.170058, 0.147555, 0.28905},
.activation = Options::SOFTMAX,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 2,
.max_abs_diff = 1e-6},
}), }),
[](const testing::TestParamInfo< [](const testing::TestParamInfo<
TensorsToSegmentationCalculatorTest::ParamType>& info) { TensorsToSegmentationCalculatorTest::ParamType>& info) {

111
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc Normal file
View File

@ -0,0 +1,111 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <string>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/port/parse_text_proto.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
switch (activation) {
case TensorsToSegmentationCalculatorOptions::NONE:
return "NONE";
case TensorsToSegmentationCalculatorOptions::SIGMOID:
return "SIGMOID";
case TensorsToSegmentationCalculatorOptions::SOFTMAX:
return "SOFTMAX";
}
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array) {
std::vector<unsigned char> result;
result.reserve(array.size());
for (int i = 0; i < array.size(); ++i) {
result.push_back(static_cast<unsigned char>(array[i]));
}
return result;
}
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
std::vector<float> result;
result.reserve(values.size() * 4);
for (const float& value : values) {
result.push_back(value);
result.push_back(0);
result.push_back(0);
result.push_back(value);
}
return result;
}
// For GPU tests, the input tensor needs to be moved to GPU, using
// TensorViewRequestor. After calculation, the output needs to be moved back
// to CPU, using ToImageCalculator. The output is an ImageFrame.
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
std::string pre_process = R"pb(
node {
calculator: "mediapipe.aimatter.TensorViewRequestor"
input_stream: "TENSORS:tensors"
output_stream: "TENSORS:tensors_gpu"
options {
[mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
}
}
)pb";
std::string post_process = R"pb(
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:image_as_mask_gpu"
output_stream: "IMAGE_CPU:image_as_mask"
}
)pb";
return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size" $0
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors$1"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask$2"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $3
gpu_origin: TOP_LEFT
}
}
} $4
)pb",
test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
test_gpu ? post_process : ""));
}
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe

57
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array);
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
int rows = 1;
int cols = 1;
int rows_new = 1;
int cols_new = 1;
int channels = 1;
double max_abs_diff = 1e-7;
};
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_

50
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/port/gtest.h"
namespace mediapipe::tensors_to_segmentation_utils {
namespace {
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ActivationTypeToStringWorksCorrectly) {
EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ArrayFloatToUnsignedCharWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<unsigned char> expected = {1, 2, 3};
EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
MakeRedAlphaMatrixWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
}
} // namespace
} // namespace mediapipe::tensors_to_segmentation_utils

43
mediapipe/calculators/tensor/tensors_to_segmentation_converter.h Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#include <memory>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
class TensorsToSegmentationConverter {
public:
virtual ~TensorsToSegmentationConverter() = default;
// Converts tensors to image mask.
// Returns a unique pointer containing the converted image.
// @input_tensors contains the tensors needed to be processed.
// @output_width/height describes output dimensions to reshape the output mask
// into.
virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) = 0;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_

157
mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#include <algorithm>
#include <cmath>
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
class OpenCvProcessor : public TensorsToSegmentationConverter {
public:
absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
options_ = options;
return absl::OkStatus();
}
absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) override;
private:
template <class T>
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
TensorsToSegmentationCalculatorOptions options_;
};
absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) {
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
auto output_mask = std::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
return output_mask;
}
template <class T>
absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
} // namespace
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
auto converter = std::make_unique<OpenCvProcessor>();
MP_RETURN_IF_ERROR(converter->Init(options));
return converter;
}
} // namespace mediapipe

31
mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#include <memory>
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
namespace mediapipe {
// Creates OpenCV tensors-to-segmentation converter.
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(
const mediapipe::TensorsToSegmentationCalculatorOptions& options);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_

52
mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc Normal file
View File

@ -0,0 +1,52 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
int NumGroups(int size, int group_size) {
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace mediapipe

34
mediapipe/calculators/tensor/tensors_to_segmentation_utils.h Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
namespace mediapipe {
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size); // NOLINT
bool CanUseGpu();
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_

63
mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc Normal file
View File

@ -0,0 +1,63 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
using ::testing::HasSubstr;
TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
EXPECT_EQ(NumGroups(13, 4), 4);
EXPECT_EQ(NumGroups(4, 13), 1);
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
std::vector<int> dims_3 = {2, 3, 4};
absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
MP_ASSERT_OK(result_1);
EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
std::vector<int> dims_4 = {1, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
MP_ASSERT_OK(result_2);
EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
std::vector<int> dims_4 = {2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Expected batch to be 1 for BHWC heatmap"));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
std::vector<int> dims_5 = {1, 2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Invalid shape for segmentation tensor"));
}
} // namespace
} // namespace mediapipe

2
mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
View File

@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
// and label and label_id are optional but at least one of them should be set. // and label and label_id are optional but at least one of them should be set.
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store // "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
// prefixed versions of each stream, which allows for multiple image streams to // prefixed versions of each stream, which allows for multiple image streams to
// be included. However, the default names are suppored by more tools. // be included. However, the default names are supported by more tools.
// //
// Example config: // Example config:
// node { // node {

4
mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc
View File

@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
// -- 1-D or 2-D Tensor // -- 1-D or 2-D Tensor
// Output: // Output:
// -- Matrix with the same values as the Tensor // -- Matrix with the same values as the Tensor
// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape. // If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape. // If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
// //
// Example Config // Example Config
// node: { // node: {

2
mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc
View File

@ -15,9 +15,9 @@
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float> // Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>. // OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
#include <cstdint>
#include <memory> #include <memory>
#include "absl/base/integral_types.h"
#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"

6
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
View File

@ -111,8 +111,8 @@ class InferenceState {
// input_side_packet. // input_side_packet.
// //
// The input and output streams are TensorFlow tensors labeled by tags. The tags // The input and output streams are TensorFlow tensors labeled by tags. The tags
// for the streams are matched to feeds and fetchs in a TensorFlow session using // for the streams are matched to feeds and fetches in a TensorFlow session
// a named_signature.generic_signature in the ModelManifest. The // using a named_signature.generic_signature in the ModelManifest. The
// generic_signature is used as key-value pairs between the MediaPipe tag and // generic_signature is used as key-value pairs between the MediaPipe tag and
// the TensorFlow tensor. The signature_name in the options proto determines // the TensorFlow tensor. The signature_name in the options proto determines
// which named_signature is used. The keys in the generic_signature must be // which named_signature is used. The keys in the generic_signature must be
@ -128,7 +128,7 @@ class InferenceState {
// addition. Once batch_size inputs have been provided, the batch will be run // addition. Once batch_size inputs have been provided, the batch will be run
// and the output tensors sent out on the output streams with timestamps // and the output tensors sent out on the output streams with timestamps
// corresponding to the input stream packets. Setting the batch_size to 1 // corresponding to the input stream packets. Setting the batch_size to 1
// completely disables batching, but is indepdent of add_batch_dim_to_tensors. // completely disables batching, but is independent of add_batch_dim_to_tensors.
// //
// The TensorFlowInferenceCalculator also support feeding states recurrently for // The TensorFlowInferenceCalculator also support feeding states recurrently for
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the // RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the

2
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto
View File

@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
// If the 0th dimension is the batch dimension, then the tensors are // If the 0th dimension is the batch dimension, then the tensors are
// concatenated on that dimension. If the 0th is a data dimension, then a 0th // concatenated on that dimension. If the 0th is a data dimension, then a 0th
// dimension is added before concatenating. If added, the extra dimension is // dimension is added before concatenating. If added, the extra dimension is
// removed before outputing the tensor. Examples of each case: If you want // removed before outputting the tensor. Examples of each case: If you want
// to batch spectra of audio over time for an LSTM, a time-frequency // to batch spectra of audio over time for an LSTM, a time-frequency
// representation has a 0th dimension as the batch dimension. If you want to // representation has a 0th dimension as the batch dimension. If you want to
// batch frames of video that are [width, height, channels], the batch // batch frames of video that are [width, height, channels], the batch

2
mediapipe/calculators/tflite/testdata/README.md vendored
View File

@ -1,2 +1,2 @@
The model files add.bin, add_quantized.bin The model files add.bin, add_quantized.bin
(and corresponding metatada json files) come from tensorflow/lite/testdata/ (and corresponding metadata json files) come from tensorflow/lite/testdata/

2
mediapipe/calculators/tflite/tflite_converter_calculator.cc
View File

@ -95,7 +95,7 @@ struct GPUData {
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer // into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
// or MTLBuffer. // or MTLBuffer.
// //
// This calculator is designed to be used with the TfLiteInferenceCalcualtor, // This calculator is designed to be used with the TfLiteInferenceCalculator,
// as a pre-processing step for calculator inputs. // as a pre-processing step for calculator inputs.
// //
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1], // IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],

2
mediapipe/calculators/tflite/tflite_converter_calculator.proto
View File

@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
// Custom settings to override the internal scaling factors `div` and `sub`. // Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on // Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom // CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overriden, and // values take effect, the |zero_center| setting above will be overridden, and
// the normalized_value will be calculated as: // the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub. // normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false]; optional bool use_custom_normalization = 6 [default = false];

2
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto
View File

@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463; optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
} }
// Score threshold for perserving the class. // Score threshold for preserving the class.
optional float min_score_threshold = 1; optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then // Number of highest scoring labels to output. If top_k is not positive then
// all labels are used. // all labels are used.

2
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
View File

@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
// tensors can have 2 or 3 tensors. First tensor is the predicted // tensors can have 2 or 3 tensors. First tensor is the predicted
// raw boxes/keypoints. The size of the values must be (num_boxes // raw boxes/keypoints. The size of the values must be (num_boxes
// * num_predicted_values). Second tensor is the score tensor. The // * num_predicted_values). Second tensor is the score tensor. The
// size of the valuse must be (num_boxes * num_classes). It's // size of the values must be (num_boxes * num_classes). It's
// optional to pass in a third tensor for anchors (e.g. for SSD // optional to pass in a third tensor for anchors (e.g. for SSD
// models) depend on the outputs of the detection model. The size // models) depend on the outputs of the detection model. The size
// of anchor tensor must be (num_boxes * 4). // of anchor tensor must be (num_boxes * 4).

2
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto
View File

@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
// representation has a bottom-left origin (e.g., in OpenGL). // representation has a bottom-left origin (e.g., in OpenGL).
optional bool flip_vertically = 18 [default = false]; optional bool flip_vertically = 18 [default = false];
// Score threshold for perserving decoded detections. // Score threshold for preserving decoded detections.
optional float min_score_thresh = 19; optional float min_score_thresh = 19;
} }

2
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
View File

@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
RET_CHECK(options_.has_input_image_height() && RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width()) options_.has_input_image_width())
<< "Must provide input width/height for using flip_vertically option " << "Must provide input width/height for using flip_vertically option "
"when outputing landmarks in absolute coordinates."; "when outputting landmarks in absolute coordinates.";
} }
flip_horizontally_ = flip_horizontally_ =

BIN
mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

3
mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,7 @@
distributionBase=GRADLE_USER_HOME distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
networkTimeout=10000 networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists zipStorePath=wrapper/dists

29
mediapipe/examples/android/solutions/gradlew vendored
View File

@ -83,10 +83,8 @@ done
# This is normally unused # This is normally unused
# shellcheck disable=SC2034 # shellcheck disable=SC2034
APP_BASE_NAME=${0##*/} APP_BASE_NAME=${0##*/}
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value. # Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum MAX_FD=maximum
@ -133,18 +131,21 @@ location of your Java installation."
fi fi
else else
JAVACMD=java JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the Please set the JAVA_HOME variable in your environment to match the
location of your Java installation." location of your Java installation."
fi fi
fi
# Increase the maximum file descriptors if we can. # Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #( case $MAX_FD in #(
max*) max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC3045 # shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) || MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit" warn "Could not query maximum file descriptor limit"
esac esac
@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
'' | soft) :;; #( '' | soft) :;; #(
*) *)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC3045 # shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" || ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD" warn "Could not set maximum file descriptor limit to $MAX_FD"
esac esac
@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
done done
fi fi
# Collect all arguments for the java command;
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
# shell script including quotes and variable substitutions, so put them in DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# double quotes to make sure that they get re-expanded; and
# * put everything else in single quotes, so that it's not re-expanded. # Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \ set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \ "-Dorg.gradle.appname=$APP_BASE_NAME" \

2
mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc
View File

@ -55,7 +55,7 @@ absl::Status RunMPPGraph() {
for (const std::string& kv_pair : kv_pairs) { for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '='); std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2); RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); RET_CHECK(!input_side_packets.contains(name_and_value[0]));
std::string input_side_packet_contents; std::string input_side_packet_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents( MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
name_and_value[1], &input_side_packet_contents)); name_and_value[1], &input_side_packet_contents));

2
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
View File

@ -56,7 +56,7 @@ absl::Status RunMPPGraph() {
for (const std::string& kv_pair : kv_pairs) { for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '='); std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2); RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); RET_CHECK(!input_side_packets.contains(name_and_value[0]));
std::string input_side_packet_contents; std::string input_side_packet_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents( MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
name_and_value[1], &input_side_packet_contents)); name_and_value[1], &input_side_packet_contents));

16
mediapipe/framework/api2/builder.h
View File

@ -330,6 +330,14 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
class NodeBase { class NodeBase {
public: public:
NodeBase() = default;
~NodeBase() = default;
NodeBase(NodeBase&&) = default;
NodeBase& operator=(NodeBase&&) = default;
// Explicitly delete copies to improve error messages.
NodeBase(const NodeBase&) = delete;
NodeBase& operator=(const NodeBase&) = delete;
// TODO: right now access to an indexed port is made directly by // TODO: right now access to an indexed port is made directly by
// specifying both a tag and an index. It would be better to represent this // specifying both a tag and an index. It would be better to represent this
// as a two-step lookup, first getting a multi-port, and then accessing one // as a two-step lookup, first getting a multi-port, and then accessing one
@ -585,6 +593,14 @@ class PacketGenerator {
class Graph { class Graph {
public: public:
Graph() = default;
~Graph() = default;
Graph(Graph&&) = default;
Graph& operator=(Graph&&) = default;
// Explicitly delete copies to improve error messages.
Graph(const Graph&) = delete;
Graph& operator=(const Graph&) = delete;
void SetType(std::string type) { type_ = std::move(type); } void SetType(std::string type) { type_ = std::move(type); }
// Creates a node of a specific type. Should be used for calculators whose // Creates a node of a specific type. Should be used for calculators whose

9
mediapipe/framework/formats/BUILD
View File

@ -124,6 +124,15 @@ cc_library(
], ],
) )
cc_library(
name = "ahwb_view",
hdrs = ["ahwb_view.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/gpu:gpu_buffer_storage",
],
)
cc_library( cc_library(
name = "affine_transform", name = "affine_transform",
srcs = ["affine_transform.cc"], srcs = ["affine_transform.cc"],

54
mediapipe/framework/formats/ahwb_view.h Normal file
View File

@ -0,0 +1,54 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#include "mediapipe/framework/port.h"
#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
#include <android/hardware_buffer.h>
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
// Intended use cases:
// - Extracting an AHWB for processing in another library after it's produced by
// MediaPipe.
// - Sending AHWBs to compute devices that are able to map the memory for their
// own usage.
// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
// other CPU/GPU uses of AHWBs.
class AhwbView {
public:
explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
// Non-copyable
AhwbView(const AhwbView&) = delete;
AhwbView& operator=(const AhwbView&) = delete;
// Non-movable
AhwbView(AhwbView&&) = delete;
// Only supports synchronous usage. All users of GetHandle must finish
// accessing the buffer before this view object is destroyed to avoid race
// conditions.
// TODO: Support asynchronous usage.
const AHardwareBuffer* GetHandle() const { return handle_; }
private:
const AHardwareBuffer* handle_;
};
namespace internal {
// Makes this class available as a GpuBuffer view.
template <>
class ViewProvider<AhwbView> {
public:
virtual ~ViewProvider() = default;
virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
virtual AhwbView GetWriteView(types<AhwbView>) = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_

7
mediapipe/framework/port.h
View File

@ -50,7 +50,7 @@
// but may or may not still be able to run other OpenGL code. // but may or may not still be able to run other OpenGL code.
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \ #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
(defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \ (defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
MEDIAPIPE_USING_SWIFTSHADER) MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
#define MEDIAPIPE_DISABLE_GL_COMPUTE #define MEDIAPIPE_DISABLE_GL_COMPUTE
#endif #endif
@ -104,4 +104,9 @@
#endif #endif
#endif // MEDIAPIPE_HAS_RTTI #endif // MEDIAPIPE_HAS_RTTI
// AHardware buffers are only available since Android API 26.
#if (__ANDROID_API__ >= 26)
#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
#endif
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_ #endif // MEDIAPIPE_FRAMEWORK_PORT_H_

3
mediapipe/framework/tool/BUILD
View File

@ -616,6 +616,7 @@ cc_test(
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/functional:bind_front",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
) )
@ -856,6 +857,7 @@ cc_library(
mediapipe_cc_test( mediapipe_cc_test(
name = "switch_demux_calculator_test", name = "switch_demux_calculator_test",
srcs = ["switch_demux_calculator_test.cc"], srcs = ["switch_demux_calculator_test.cc"],
requires_full_emulation = False,
deps = [ deps = [
":container_util", ":container_util",
":switch_demux_calculator", ":switch_demux_calculator",
@ -891,6 +893,7 @@ cc_library(
mediapipe_cc_test( mediapipe_cc_test(
name = "switch_mux_calculator_test", name = "switch_mux_calculator_test",
srcs = ["switch_mux_calculator_test.cc"], srcs = ["switch_mux_calculator_test.cc"],
requires_full_emulation = False,
deps = [ deps = [
":container_util", ":container_util",
":switch_mux_calculator", ":switch_mux_calculator",

1
mediapipe/framework/tool/sink_test.cc
View File

@ -17,6 +17,7 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include "absl/functional/bind_front.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"

4
mediapipe/framework/tool/validate_name.cc
View File

@ -134,7 +134,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
RET_CHECK(name); RET_CHECK(name);
absl::Status tag_status = absl::OkStatus(); absl::Status tag_status = absl::OkStatus();
absl::Status name_status = absl::UnknownError(""); absl::Status name_status = absl::UnknownError("");
int name_index = 0; int name_index = -1;
std::vector<std::string> v = absl::StrSplit(tag_and_name, ':'); std::vector<std::string> v = absl::StrSplit(tag_and_name, ':');
if (v.size() == 1) { if (v.size() == 1) {
name_status = ValidateName(v[0]); name_status = ValidateName(v[0]);
@ -143,7 +143,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
tag_status = ValidateTag(v[0]); tag_status = ValidateTag(v[0]);
name_status = ValidateName(v[1]); name_status = ValidateName(v[1]);
name_index = 1; name_index = 1;
} } // else omitted, name_index == -1, triggering error.
if (name_index == -1 || tag_status != absl::OkStatus() || if (name_index == -1 || tag_status != absl::OkStatus() ||
name_status != absl::OkStatus()) { name_status != absl::OkStatus()) {
tag->clear(); tag->clear();

38
mediapipe/gpu/BUILD
View File

@ -511,11 +511,19 @@ cc_library(
], ],
}), }),
deps = [ deps = [
":gl_base_hdr",
":gl_context",
":gl_texture_buffer", ":gl_texture_buffer",
":gl_texture_view",
":gpu_buffer_format", ":gpu_buffer_format",
":gpu_buffer_storage", ":gpu_buffer_storage",
":image_frame_view", ":image_frame_view",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:ahwb_view",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//third_party/GL:EGL_headers",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
], ],
) )
@ -526,12 +534,14 @@ mediapipe_proto_library(
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
objc_library( cc_library(
name = "pixel_buffer_pool_util", name = "pixel_buffer_pool_util",
srcs = ["pixel_buffer_pool_util.mm"], srcs = ["pixel_buffer_pool_util.cc"],
hdrs = ["pixel_buffer_pool_util.h"], hdrs = ["pixel_buffer_pool_util.h"],
copts = [ copts = [
"-x objective-c++",
"-Wno-shorten-64-to-32", "-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
], ],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
@ -542,13 +552,14 @@ objc_library(
], ],
) )
objc_library( cc_library(
name = "metal_shared_resources", name = "metal_shared_resources",
srcs = ["metal_shared_resources.mm"], srcs = ["metal_shared_resources.cc"],
hdrs = ["metal_shared_resources.h"], hdrs = ["metal_shared_resources.h"],
copts = [ copts = [
"-x objective-c++", "-x objective-c++",
"-Wno-shorten-64-to-32", "-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
], ],
features = ["-layering_check"], features = ["-layering_check"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
@ -557,15 +568,17 @@ objc_library(
"@google_toolbox_for_mac//:GTM_Defines", "@google_toolbox_for_mac//:GTM_Defines",
] + [ ] + [
], ],
alwayslink = 1,
) )
objc_library( cc_library(
name = "MPPMetalUtil", name = "MPPMetalUtil",
srcs = ["MPPMetalUtil.mm"], srcs = ["MPPMetalUtil.cc"],
hdrs = ["MPPMetalUtil.h"], hdrs = ["MPPMetalUtil.h"],
copts = [ copts = [
"-x objective-c++", "-x objective-c++",
"-Wno-shorten-64-to-32", "-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
], ],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
@ -575,6 +588,7 @@ objc_library(
"@com_google_absl//absl/time", "@com_google_absl//absl/time",
"@google_toolbox_for_mac//:GTM_Defines", "@google_toolbox_for_mac//:GTM_Defines",
], ],
alwayslink = 1,
) )
mediapipe_proto_library( mediapipe_proto_library(
@ -655,6 +669,8 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/status",
] + select({ ] + select({
"//conditions:default": [], "//conditions:default": [],
"//mediapipe:apple": [ "//mediapipe:apple": [
@ -857,12 +873,14 @@ cc_library(
}), }),
) )
objc_library( cc_library(
name = "MPPMetalHelper", name = "MPPMetalHelper",
srcs = ["MPPMetalHelper.mm"], srcs = ["MPPMetalHelper.cc"],
hdrs = ["MPPMetalHelper.h"], hdrs = ["MPPMetalHelper.h"],
copts = [ copts = [
"-Wno-shorten-64-to-32", "-Wno-shorten-64-to-32",
"-x objective-c++",
"-fobjc-arc",
], ],
features = ["-layering_check"], features = ["-layering_check"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
@ -1215,9 +1233,13 @@ mediapipe_cc_test(
], ],
requires_full_emulation = True, requires_full_emulation = True,
deps = [ deps = [
":gl_texture_buffer",
":gl_texture_util",
":gpu_buffer_format", ":gpu_buffer_format",
":gpu_buffer_storage_ahwb", ":gpu_buffer_storage_ahwb",
":gpu_test_base",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/tool:test_util",
], ],
) )

60
mediapipe/gpu/MPPMetalHelper.mm → mediapipe/gpu/MPPMetalHelper.cc
View File

@ -14,15 +14,14 @@
#import "mediapipe/gpu/MPPMetalHelper.h" #import "mediapipe/gpu/MPPMetalHelper.h"
#import "GTMDefines.h"
#include "absl/log/absl_check.h" #include "absl/log/absl_check.h"
#include "absl/log/absl_log.h" #include "absl/log/absl_log.h"
#include "mediapipe/framework/port/ret_check.h"
#import "mediapipe/gpu/gpu_buffer.h" #import "mediapipe/gpu/gpu_buffer.h"
#import "mediapipe/gpu/gpu_service.h" #import "mediapipe/gpu/gpu_service.h"
#import "mediapipe/gpu/graph_support.h" #import "mediapipe/gpu/graph_support.h"
#import "mediapipe/gpu/metal_shared_resources.h" #import "mediapipe/gpu/metal_shared_resources.h"
#import "GTMDefines.h"
#include "mediapipe/framework/port/ret_check.h"
@interface MPPMetalHelper () { @interface MPPMetalHelper () {
mediapipe::GpuResources* _gpuResources; mediapipe::GpuResources* _gpuResources;
@ -31,7 +30,8 @@
namespace mediapipe { namespace mediapipe {
// Using a C++ class so it can be declared as a friend of LegacyCalculatorSupport. // Using a C++ class so it can be declared as a friend of
// LegacyCalculatorSupport.
class MetalHelperLegacySupport { class MetalHelperLegacySupport {
public: public:
static CalculatorContract* GetCalculatorContract() { static CalculatorContract* GetCalculatorContract() {
@ -61,7 +61,8 @@ class MetalHelperLegacySupport {
- (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc { - (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc {
if (!cc) return nil; if (!cc) return nil;
return [self initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()]; return [self
initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
} }
+ (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc { + (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc {
@ -77,7 +78,8 @@ class MetalHelperLegacySupport {
} }
// Legacy support. // Legacy support.
- (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets { - (instancetype)initWithSidePackets:
(const mediapipe::PacketSet&)inputSidePackets {
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext(); auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext();
if (cc) { if (cc) {
ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets()); ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets());
@ -85,16 +87,19 @@ class MetalHelperLegacySupport {
} }
// TODO: remove when we can. // TODO: remove when we can.
ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses " ABSL_LOG(WARNING)
<< "CalculatorContext not available. If this calculator uses "
"CalculatorBase, call initWithCalculatorContext instead."; "CalculatorBase, call initWithCalculatorContext instead.";
mediapipe::GpuSharedData* gpu_shared = mediapipe::GpuSharedData* gpu_shared =
inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get<mediapipe::GpuSharedData*>(); inputSidePackets.Tag(mediapipe::kGpuSharedTagName)
.Get<mediapipe::GpuSharedData*>();
return [self initWithGpuResources:gpu_shared->gpu_resources.get()]; return [self initWithGpuResources:gpu_shared->gpu_resources.get()];
} }
// Legacy support. // Legacy support.
+ (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets { + (absl::Status)setupInputSidePackets:
(mediapipe::PacketTypeSet*)inputSidePackets {
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract(); auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract();
if (cc) { if (cc) {
ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets()); ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets());
@ -102,11 +107,11 @@ class MetalHelperLegacySupport {
} }
// TODO: remove when we can. // TODO: remove when we can.
ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this " ABSL_LOG(WARNING)
<< "CalculatorContract not available. If you're calling this "
"from a GetContract method, call updateContract instead."; "from a GetContract method, call updateContract instead.";
auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0); auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0);
RET_CHECK(id.IsValid()) RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName
<< "A " << mediapipe::kGpuSharedTagName
<< " input side packet is required here."; << " input side packet is required here.";
inputSidePackets->Get(id).Set<mediapipe::GpuSharedData*>(); inputSidePackets->Get(id).Set<mediapipe::GpuSharedData*>();
return absl::OkStatus(); return absl::OkStatus();
@ -125,10 +130,12 @@ class MetalHelperLegacySupport {
} }
- (id<MTLCommandBuffer>)commandBuffer { - (id<MTLCommandBuffer>)commandBuffer {
return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer]; return
[_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
} }
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer - (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer
plane:(size_t)plane { plane:(size_t)plane {
CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer); CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer);
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
@ -178,40 +185,47 @@ class MetalHelperLegacySupport {
CVMetalTextureRef texture; CVMetalTextureRef texture;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage( CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
NULL, _gpuResources->metal_shared().resources().mtlTextureCache, NULL, _gpuResources->metal_shared().resources().mtlTextureCache,
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane, mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width,
&texture); height, plane, &texture);
ABSL_CHECK_EQ(err, kCVReturnSuccess); ABSL_CHECK_EQ(err, kCVReturnSuccess);
return texture; return texture;
} }
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer { - (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer {
return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0]; return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0];
} }
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer { - (id<MTLTexture>)metalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer {
return [self metalTextureWithGpuBuffer:gpuBuffer plane:0]; return [self metalTextureWithGpuBuffer:gpuBuffer plane:0];
} }
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer - (id<MTLTexture>)metalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer
plane:(size_t)plane { plane:(size_t)plane {
CFHolder<CVMetalTextureRef> cvTexture; CFHolder<CVMetalTextureRef> cvTexture;
cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]); cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]);
return CVMetalTextureGetTexture(*cvTexture); return CVMetalTextureGetTexture(*cvTexture);
} }
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height { - (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
height:(int)height {
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height); return _gpuResources->gpu_buffer_pool().GetBuffer(width, height);
} }
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width - (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
height:(int)height height:(int)height
format:(mediapipe::GpuBufferFormat)format { format:(mediapipe::GpuBufferFormat)
format {
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format); return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format);
} }
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error { - (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name
error:(NSError* _Nullable*)error {
return [_gpuResources->metal_shared().resources().mtlDevice return [_gpuResources->metal_shared().resources().mtlDevice
newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name newLibraryWithFile:[[NSBundle bundleForClass:[self class]]
pathForResource:name
ofType:@"metallib"] ofType:@"metallib"]
error:error]; error:error];
} }

8
mediapipe/gpu/MPPMetalUtil.mm → mediapipe/gpu/MPPMetalUtil.cc
View File

@ -69,10 +69,10 @@
while (!bufferCompleted) { while (!bufferCompleted) {
auto duration = absl::Now() - start_time; auto duration = absl::Now() - start_time;
// If the spin-lock takes more than 5 ms then go to blocking wait: // If the spin-lock takes more than 5 ms then go to blocking wait:
// - it frees the CPU core for another threads: increase the performance/decrease power // - it frees the CPU core for another threads: increase the
// consumption. // performance/decrease power consumption.
// - if a driver thread that notifies that the GPU buffer is completed has lower priority then // - if a driver thread that notifies that the GPU buffer is completed has
// the CPU core is allocated for the thread. // lower priority then the CPU core is allocated for the thread.
if (duration >= absl::Milliseconds(5)) { if (duration >= absl::Milliseconds(5)) {
[commandBuffer waitUntilCompleted]; [commandBuffer waitUntilCompleted];
break; break;

4
mediapipe/gpu/gl_calculator_helper.cc
View File

@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
// static // static
absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc, absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
bool requesst_gpu_as_optional) { bool request_gpu_as_optional) {
if (requesst_gpu_as_optional) { if (request_gpu_as_optional) {
cc->UseService(kGpuService).Optional(); cc->UseService(kGpuService).Optional();
} else { } else {
cc->UseService(kGpuService); cc->UseService(kGpuService);

2
mediapipe/gpu/gl_calculator_helper.h
View File

@ -68,7 +68,7 @@ class GlCalculatorHelper {
// This method can be called from GetContract to set up the needed GPU // This method can be called from GetContract to set up the needed GPU
// resources. // resources.
static absl::Status UpdateContract(CalculatorContract* cc, static absl::Status UpdateContract(CalculatorContract* cc,
bool requesst_gpu_as_optional = false); bool request_gpu_as_optional = false);
// This method can be called from FillExpectations to set the correct types // This method can be called from FillExpectations to set the correct types
// for the shared GL input side packet(s). // for the shared GL input side packet(s).

9
mediapipe/gpu/gl_texture_buffer.cc
View File

@ -14,6 +14,8 @@
#include "mediapipe/gpu/gl_texture_buffer.h" #include "mediapipe/gpu/gl_texture_buffer.h"
#include <cstdint>
#include "absl/log/absl_check.h" #include "absl/log/absl_check.h"
#include "absl/log/absl_log.h" #include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
@ -131,6 +133,13 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
SymbolAvailable(&glTexStorage2D)) { SymbolAvailable(&glTexStorage2D)) {
ABSL_CHECK(data == nullptr) << "unimplemented"; ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else if (info.immutable) {
ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
context->GetGlVersion() != GlVersion::kGLES2)
<< "Immutable GpuBuffer format requested is not supported in this "
<< "GlContext. Format was " << static_cast<uint32_t>(format_);
ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else { } else {
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_, glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
height_, 0 /* border */, info.gl_format, info.gl_type, data); height_, 0 /* border */, info.gl_format, info.gl_type, data);

1
mediapipe/gpu/gl_texture_buffer.h
View File

@ -19,6 +19,7 @@
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_ #define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
#include <atomic> #include <atomic>
#include <memory>
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"

2
mediapipe/gpu/gl_thread_collector.h
View File

@ -17,7 +17,7 @@
#include <cstdlib> #include <cstdlib>
#if defined(MEDIAPIPE_USING_SWIFTSHADER) #if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1 #define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
#endif #endif

16
mediapipe/gpu/gpu_buffer_format.cc
View File

@ -35,6 +35,10 @@ namespace mediapipe {
#endif // GL_HALF_FLOAT_OES #endif // GL_HALF_FLOAT_OES
#endif // __EMSCRIPTEN__ #endif // __EMSCRIPTEN__
#ifndef GL_RGBA8
#define GL_RGBA8 0x8058
#endif // GL_RGBA8
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#ifdef GL_ES_VERSION_2_0 #ifdef GL_ES_VERSION_2_0
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) { static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
@ -163,6 +167,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
{ {
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1}, {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
}}, }},
{GpuBufferFormat::kImmutableRGBAFloat128,
{
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
}},
{GpuBufferFormat::kImmutableRGBA32,
{
{GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
}},
}}; }};
static const auto* gles2_format_info = ([] { static const auto* gles2_format_info = ([] {
@ -206,6 +218,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) { switch (format) {
case GpuBufferFormat::kImmutableRGBA32:
case GpuBufferFormat::kBGRA32: case GpuBufferFormat::kBGRA32:
// TODO: verify we are handling order of channels correctly. // TODO: verify we are handling order of channels correctly.
return ImageFormat::SRGBA; return ImageFormat::SRGBA;
@ -221,10 +234,11 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
return ImageFormat::SRGB; return ImageFormat::SRGB;
case GpuBufferFormat::kTwoComponentFloat32: case GpuBufferFormat::kTwoComponentFloat32:
return ImageFormat::VEC32F2; return ImageFormat::VEC32F2;
case GpuBufferFormat::kImmutableRGBAFloat128:
case GpuBufferFormat::kRGBAFloat128: case GpuBufferFormat::kRGBAFloat128:
return ImageFormat::VEC32F4; return ImageFormat::VEC32F4;
case GpuBufferFormat::kRGBA32: case GpuBufferFormat::kRGBA32:
// TODO: this likely maps to ImageFormat::SRGBA return ImageFormat::SRGBA;
case GpuBufferFormat::kGrayHalf16: case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Alpha:
case GpuBufferFormat::kOneComponent8Red: case GpuBufferFormat::kOneComponent8Red:

9
mediapipe/gpu/gpu_buffer_format.h
View File

@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t {
kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible. kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible.
kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'), kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'),
kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'), kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
// Immutable version of kRGBA32
kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'),
// Immutable version of kRGBAFloat128
kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'),
// 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling. // 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling.
kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'), kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'),
// 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling. // 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling.
@ -78,6 +82,9 @@ struct GlTextureInfo {
// For multiplane buffers, this represents how many times smaller than // For multiplane buffers, this represents how many times smaller than
// the nominal image size a plane is. // the nominal image size a plane is.
int downscale; int downscale;
// For GLES3.1+ compute shaders, users may explicitly request immutable
// textures.
bool immutable = false;
}; };
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
return kCVPixelFormatType_64RGBAHalf; return kCVPixelFormatType_64RGBAHalf;
case GpuBufferFormat::kRGBAFloat128: case GpuBufferFormat::kRGBAFloat128:
return kCVPixelFormatType_128RGBAFloat; return kCVPixelFormatType_128RGBAFloat;
case GpuBufferFormat::kImmutableRGBA32:
case GpuBufferFormat::kImmutableRGBAFloat128:
case GpuBufferFormat::kNV12: case GpuBufferFormat::kNV12:
case GpuBufferFormat::kNV21: case GpuBufferFormat::kNV21:
case GpuBufferFormat::kI420: case GpuBufferFormat::kI420:

2
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
View File

@ -151,7 +151,7 @@ static std::shared_ptr<GpuBufferStorageCvPixelBuffer> ConvertFromImageFrame(
std::shared_ptr<GpuBufferStorageImageFrame> frame) { std::shared_ptr<GpuBufferStorageImageFrame> frame) {
auto status_or_buffer = auto status_or_buffer =
CreateCVPixelBufferForImageFrame(frame->image_frame()); CreateCVPixelBufferForImageFrame(frame->image_frame());
ABSL_CHECK(status_or_buffer.ok()); ABSL_CHECK_OK(status_or_buffer);
return std::make_shared<GpuBufferStorageCvPixelBuffer>( return std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(status_or_buffer).value()); std::move(status_or_buffer).value());
} }

3
mediapipe/gpu/gpu_origin.proto
View File

@ -16,6 +16,9 @@ syntax = "proto3";
package mediapipe; package mediapipe;
option java_package = "com.google.mediapipe.gpu";
option java_outer_classname = "GpuOriginProto";
message GpuOrigin { message GpuOrigin {
enum Mode { enum Mode {
DEFAULT = 0; DEFAULT = 0;

55
mediapipe/gpu/gpu_shared_data_internal.cc
View File

@ -14,10 +14,14 @@
#include "mediapipe/gpu/gpu_shared_data_internal.h" #include "mediapipe/gpu/gpu_shared_data_internal.h"
#include <memory>
#include <utility>
#include "absl/base/attributes.h" #include "absl/base/attributes.h"
#include "absl/log/absl_check.h" #include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/status/status.h"
#include "mediapipe/framework/deps/no_destructor.h" #include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gl_context.h" #include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_options.pb.h" #include "mediapipe/gpu/gl_context_options.pb.h"
#include "mediapipe/gpu/graph_support.h" #include "mediapipe/gpu/graph_support.h"
@ -83,8 +87,25 @@ GpuResources::StatusOrGpuResources GpuResources::Create(
} }
GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context) GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
: gl_key_context_(new GlContextMapType(),
[](auto* map) {
// This flushes all pending jobs in all GL contexts,
// ensuring that all GL contexts not referenced
// elsewhere are destroyed as part of this destructor.
// Failure to do this may cause GL threads to outlast
// this destructor and execute jobs after the
// GpuResources object is destroyed.
for (auto& [key, context] : *map) {
const auto status = std::move(context)->Run(
[]() { return absl::OkStatus(); });
ABSL_LOG_IF(ERROR, !status.ok())
<< "Failed to flush GlContext jobs: " << status;
}
delete map;
})
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
: texture_caches_(std::make_shared<CvTextureCacheManager>()), ,
texture_caches_(std::make_shared<CvTextureCacheManager>()),
gpu_buffer_pool_( gpu_buffer_pool_(
[tc = texture_caches_](const internal::GpuBufferSpec& spec, [tc = texture_caches_](const internal::GpuBufferSpec& spec,
const MultiPoolOptions& options) { const MultiPoolOptions& options) {
@ -92,7 +113,7 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
}) })
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
{ {
gl_key_context_[SharedContextKey()] = gl_context; gl_key_context_->insert({SharedContextKey(), gl_context});
named_executors_[kGpuExecutorName] = named_executors_[kGpuExecutorName] =
std::make_shared<GlContextExecutor>(gl_context.get()); std::make_shared<GlContextExecutor>(gl_context.get());
#if __APPLE__ #if __APPLE__
@ -104,6 +125,15 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
} }
GpuResources::~GpuResources() { GpuResources::~GpuResources() {
// This flushes all pending jobs in all GL contexts,
// ensuring that all existing jobs, which may refer GpuResource and kept their
// gpu resources (e.g. GpuResources::gpu_buffer_pool_) through a raw pointer,
// have finished before kept gpu resources get deleted.
for (auto& [key, context] : *gl_key_context_) {
const auto status = context->Run([]() { return absl::OkStatus(); });
ABSL_LOG_IF(ERROR, !status.ok())
<< "Failed to flush GlContext jobs: " << status;
}
#if __APPLE__ #if __APPLE__
// Note: on Apple platforms, this object contains Objective-C objects. // Note: on Apple platforms, this object contains Objective-C objects.
// The destructor will release them, but ARC must be on. // The destructor will release them, but ARC must be on.
@ -111,7 +141,7 @@ GpuResources::~GpuResources() {
#error This file must be built with ARC. #error This file must be built with ARC.
#endif #endif
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
for (auto& kv : gl_key_context_) { for (auto& kv : *gl_key_context_) {
texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache()); texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache());
} }
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@ -173,23 +203,24 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
const std::shared_ptr<GlContext>& GpuResources::gl_context( const std::shared_ptr<GlContext>& GpuResources::gl_context(
CalculatorContext* cc) { CalculatorContext* cc) {
if (cc) { if (cc) {
auto it = gl_key_context_.find(node_key_[cc->NodeName()]); auto it = gl_key_context_->find(node_key_[cc->NodeName()]);
if (it != gl_key_context_.end()) { if (it != gl_key_context_->end()) {
return it->second; return it->second;
} }
} }
return gl_key_context_[SharedContextKey()]; return gl_key_context_->at(SharedContextKey());
} }
GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext( GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext(
const std::string& key) { const std::string& key) {
auto it = gl_key_context_.find(key); auto it = gl_key_context_->find(key);
if (it == gl_key_context_.end()) { if (it == gl_key_context_->end()) {
MP_ASSIGN_OR_RETURN(std::shared_ptr<GlContext> new_context, MP_ASSIGN_OR_RETURN(
GlContext::Create(*gl_key_context_[SharedContextKey()], std::shared_ptr<GlContext> new_context,
GlContext::Create(*gl_key_context_->at(SharedContextKey()),
kGlContextUseDedicatedThread)); kGlContextUseDedicatedThread));
it = gl_key_context_.emplace(key, new_context).first; it = gl_key_context_->emplace(key, new_context).first;
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
texture_caches_->RegisterTextureCache(it->second->cv_texture_cache()); texture_caches_->RegisterTextureCache(it->second->cv_texture_cache());
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER

7
mediapipe/gpu/gpu_shared_data_internal.h
View File

@ -21,6 +21,8 @@
#ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_ #ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_ #define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#include <memory>
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_node.h" #include "mediapipe/framework/calculator_node.h"
#include "mediapipe/framework/executor.h" #include "mediapipe/framework/executor.h"
@ -82,7 +84,10 @@ class GpuResources {
const std::string& ContextKey(const std::string& canonical_node_name); const std::string& ContextKey(const std::string& canonical_node_name);
std::map<std::string, std::string> node_key_; std::map<std::string, std::string> node_key_;
std::map<std::string, std::shared_ptr<GlContext>> gl_key_context_;
using GlContextMapType = std::map<std::string, std::shared_ptr<GlContext>>;
std::unique_ptr<GlContextMapType, void (*)(GlContextMapType*)>
gl_key_context_;
#ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
std::shared_ptr<CvTextureCacheManager> texture_caches_; std::shared_ptr<CvTextureCacheManager> texture_caches_;

13
mediapipe/gpu/gpu_test_base.h
View File

@ -15,6 +15,9 @@
#ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_ #ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_
#define MEDIAPIPE_GPU_GPU_TEST_BASE_H_ #define MEDIAPIPE_GPU_GPU_TEST_BASE_H_
#include <functional>
#include <memory>
#include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
@ -22,9 +25,9 @@
namespace mediapipe { namespace mediapipe {
class GpuTestBase : public ::testing::Test { class GpuTestEnvironment {
protected: protected:
GpuTestBase() { helper_.InitializeForTest(gpu_resources_.get()); } GpuTestEnvironment() { helper_.InitializeForTest(gpu_resources_.get()); }
void RunInGlContext(std::function<void(void)> gl_func) { void RunInGlContext(std::function<void(void)> gl_func) {
helper_.RunInGlContext(std::move(gl_func)); helper_.RunInGlContext(std::move(gl_func));
@ -35,6 +38,12 @@ class GpuTestBase : public ::testing::Test {
GlCalculatorHelper helper_; GlCalculatorHelper helper_;
}; };
class GpuTestBase : public testing::Test, public GpuTestEnvironment {};
template <typename T>
class GpuTestWithParamBase : public testing::TestWithParam<T>,
public GpuTestEnvironment {};
} // namespace mediapipe } // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_TEST_BASE_H_ #endif // MEDIAPIPE_GPU_GPU_TEST_BASE_H_

2
mediapipe/gpu/metal.bzl
View File

@ -171,7 +171,7 @@ METAL_LIBRARY_ATTRS = dicts.add(apple_support.action_required_attrs(), {
metal_library = rule( metal_library = rule(
implementation = _metal_library_impl, implementation = _metal_library_impl,
attrs = METAL_LIBRARY_ATTRS, attrs = METAL_LIBRARY_ATTRS,
fragments = ["apple", "objc", "swift"], fragments = ["apple", "objc"],
output_to_genfiles = True, output_to_genfiles = True,
) )
""" """

7
mediapipe/gpu/metal_shared_resources.mm → mediapipe/gpu/metal_shared_resources.cc
View File

@ -50,9 +50,10 @@
- (CVMetalTextureCacheRef)mtlTextureCache { - (CVMetalTextureCacheRef)mtlTextureCache {
@synchronized(self) { @synchronized(self) {
if (!_mtlTextureCache) { if (!_mtlTextureCache) {
CVReturn __unused err = CVReturn __unused err = CVMetalTextureCacheCreate(
CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache); NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d ; device %@", err, NSAssert(err == kCVReturnSuccess,
@"Error at CVMetalTextureCacheCreate %d ; device %@", err,
self.mtlDevice); self.mtlDevice);
// TODO: register and flush metal caches too. // TODO: register and flush metal caches too.
} }

58
mediapipe/gpu/pixel_buffer_pool_util.mm → mediapipe/gpu/pixel_buffer_pool_util.cc
View File

@ -24,23 +24,27 @@
namespace mediapipe { namespace mediapipe {
CVPixelBufferPoolRef CreateCVPixelBufferPool( CVPixelBufferPoolRef CreateCVPixelBufferPool(int width, int height,
int width, int height, OSType pixelFormat, int keepCount, OSType pixelFormat, int keepCount,
CFTimeInterval maxAge) { CFTimeInterval maxAge) {
CVPixelBufferPoolRef pool = NULL; CVPixelBufferPoolRef pool = NULL;
NSMutableDictionary *sourcePixelBufferOptions = NSMutableDictionary *sourcePixelBufferOptions =
[(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy]; [(__bridge NSDictionary *)GetCVPixelBufferAttributesForGlCompatibility()
mutableCopy];
[sourcePixelBufferOptions addEntriesFromDictionary:@{ [sourcePixelBufferOptions addEntriesFromDictionary:@{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width), (id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height), (id)kCVPixelBufferHeightKey : @(height),
}]; }];
NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init]; NSMutableDictionary *pixelBufferPoolOptions =
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount); [[NSMutableDictionary alloc] init];
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] =
@(keepCount);
if (maxAge > 0) { if (maxAge > 0) {
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = @(maxAge); pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] =
@(maxAge);
} }
CVPixelBufferPoolCreate( CVPixelBufferPoolCreate(
@ -50,8 +54,9 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
return pool; return pool;
} }
OSStatus PreallocateCVPixelBufferPoolBuffers( OSStatus PreallocateCVPixelBufferPoolBuffers(CVPixelBufferPoolRef pool,
CVPixelBufferPoolRef pool, int count, CFDictionaryRef auxAttributes) { int count,
CFDictionaryRef auxAttributes) {
CVReturn err = kCVReturnSuccess; CVReturn err = kCVReturnSuccess;
NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init]; NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init];
for (int i = 0; i < count && err == kCVReturnSuccess; i++) { for (int i = 0; i < count && err == kCVReturnSuccess; i++) {
@ -68,30 +73,37 @@ OSStatus PreallocateCVPixelBufferPoolBuffers(
return err; return err;
} }
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(int allocationThreshold) { CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(
int allocationThreshold) {
if (allocationThreshold > 0) { if (allocationThreshold > 0) {
return (CFDictionaryRef)CFBridgingRetain( return (CFDictionaryRef)CFBridgingRetain(@{
@{(id)kCVPixelBufferPoolAllocationThresholdKey: @(allocationThreshold)}); (id)kCVPixelBufferPoolAllocationThresholdKey : @(allocationThreshold)
});
} else { } else {
return nil; return nil;
} }
} }
CVReturn CreateCVPixelBufferWithPool( CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes, CFDictionaryRef auxAttributes,
CVTextureCacheType textureCache, CVPixelBufferRef* outBuffer) { CVTextureCacheType textureCache,
return CreateCVPixelBufferWithPool(pool, auxAttributes, [textureCache](){ CVPixelBufferRef *outBuffer) {
return CreateCVPixelBufferWithPool(
pool, auxAttributes,
[textureCache]() {
#if TARGET_OS_OSX #if TARGET_OS_OSX
CVOpenGLTextureCacheFlush(textureCache, 0); CVOpenGLTextureCacheFlush(textureCache, 0);
#else #else
CVOpenGLESTextureCacheFlush(textureCache, 0); CVOpenGLESTextureCacheFlush(textureCache, 0);
#endif // TARGET_OS_OSX #endif // TARGET_OS_OSX
}, outBuffer); },
outBuffer);
} }
CVReturn CreateCVPixelBufferWithPool( CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes, CFDictionaryRef auxAttributes,
std::function<void(void)> flush, CVPixelBufferRef* outBuffer) { std::function<void(void)> flush,
CVPixelBufferRef *outBuffer) {
CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(
kCFAllocatorDefault, pool, auxAttributes, outBuffer); kCFAllocatorDefault, pool, auxAttributes, outBuffer);
if (err == kCVReturnWouldExceedAllocationThreshold) { if (err == kCVReturnWouldExceedAllocationThreshold) {
@ -103,10 +115,12 @@ CVReturn CreateCVPixelBufferWithPool(
kCFAllocatorDefault, pool, auxAttributes, outBuffer); kCFAllocatorDefault, pool, auxAttributes, outBuffer);
} }
if (err == kCVReturnWouldExceedAllocationThreshold) { if (err == kCVReturnWouldExceedAllocationThreshold) {
// TODO: allow the application to set the threshold. For now, disable it by // TODO: allow the application to set the threshold. For now, disable it
// default, since the threshold we are using is arbitrary and some graphs routinely cross it. // by default, since the threshold we are using is arbitrary and some
// graphs routinely cross it.
#ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK #ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK
NSLog(@"Using more buffers than expected! This is a debug-only warning, " NSLog(
@"Using more buffers than expected! This is a debug-only warning, "
"you can ignore it if your app works fine otherwise."); "you can ignore it if your app works fine otherwise.");
#ifdef DEBUG #ifdef DEBUG
NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description); NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description);

92
mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
View File

@ -17,7 +17,6 @@ package com.google.mediapipe.components;
import static java.lang.Math.max; import static java.lang.Math.max;
import static java.lang.Math.min; import static java.lang.Math.min;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext; import android.opengl.GLES11Ext;
import android.opengl.GLES20; import android.opengl.GLES20;
@ -25,6 +24,7 @@ import android.opengl.GLES31;
import android.opengl.GLSurfaceView; import android.opengl.GLSurfaceView;
import android.opengl.Matrix; import android.opengl.Matrix;
import android.util.Log; import android.util.Log;
import android.util.Pair;
import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.CommonShaders; import com.google.mediapipe.glutil.CommonShaders;
import com.google.mediapipe.glutil.ShaderUtil; import com.google.mediapipe.glutil.ShaderUtil;
@ -51,11 +51,9 @@ import javax.microedition.khronos.opengles.GL10;
* {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}). * {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}).
*/ */
public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
/** /** Listener for image capture requests. */
* Listener for Bitmap capture requests. public interface ImageCaptureListener {
*/ void onImageCaptured(int width, int height, int[] data);
public interface BitmapCaptureListener {
void onBitmapCaptured(Bitmap result);
} }
/** /**
@ -86,27 +84,28 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
private float[] textureTransformMatrix = new float[16]; private float[] textureTransformMatrix = new float[16];
private SurfaceTexture surfaceTexture = null; private SurfaceTexture surfaceTexture = null;
private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>(); private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean(); private final AtomicBoolean captureNextFrameImage = new AtomicBoolean();
private BitmapCaptureListener bitmapCaptureListener; private ImageCaptureListener imageCaptureListener;
// Specifies whether a black CLAMP_TO_BORDER effect should be used. // Specifies whether a black CLAMP_TO_BORDER effect should be used.
private boolean shouldClampToBorder = false; private boolean shouldClampToBorder = false;
private Scale scale = Scale.FILL; private Scale scale = Scale.FILL;
/** private float zoomFactor = 1.0f;
* Sets the {@link BitmapCaptureListener}. private Pair<Float, Float> zoomLocation = new Pair<>(0.5f, 0.5f);
*/
public void setBitmapCaptureListener(BitmapCaptureListener bitmapCaptureListener) { /** Sets the {@link ImageCaptureListener}. */
this.bitmapCaptureListener = bitmapCaptureListener; public void setImageCaptureListener(ImageCaptureListener imageCaptureListener) {
this.imageCaptureListener = imageCaptureListener;
} }
/** /**
* Request to capture Bitmap of the next frame. * Request to capture Image of the next frame.
* *
* The result will be provided to the {@link BitmapCaptureListener} if one is set. Please note * <p>The result will be provided to the {@link ImageCaptureListener} if one is set. Please note
* this is an expensive operation and the result may not be available for a while. * this is an expensive operation and the result may not be available for a while.
*/ */
public void captureNextFrameBitmap() { public void captureNextFrameImage() {
captureNextFrameBitmap.set(true); captureNextFrameImage.set(true);
} }
@Override @Override
@ -202,9 +201,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
ShaderUtil.checkGlError("glDrawArrays"); ShaderUtil.checkGlError("glDrawArrays");
// Capture Bitmap if requested. // Capture image if requested.
BitmapCaptureListener bitmapCaptureListener = this.bitmapCaptureListener; ImageCaptureListener imageCaptureListener = this.imageCaptureListener;
if (captureNextFrameBitmap.getAndSet(false) && bitmapCaptureListener != null) { if (captureNextFrameImage.getAndSet(false) && imageCaptureListener != null) {
// Find the name of the bound texture. // Find the name of the bound texture.
int[] texName = new int[1]; int[] texName = new int[1];
if (surfaceTexture != null) { if (surfaceTexture != null) {
@ -219,8 +218,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1); GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1);
int texWidth = texDims[0]; int texWidth = texDims[0];
int texHeight = texDims[1]; int texHeight = texDims[1];
int bitmapSize = texWidth * texHeight; int imageSize = texWidth * texHeight;
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(bitmapSize * 4); ByteBuffer byteBuffer = ByteBuffer.allocateDirect(imageSize * 4);
byteBuffer.order(ByteOrder.nativeOrder()); byteBuffer.order(ByteOrder.nativeOrder());
// Read pixels from texture. // Read pixels from texture.
@ -235,27 +234,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES20.glDeleteFramebuffers(1, fbo, 0); GLES20.glDeleteFramebuffers(1, fbo, 0);
ShaderUtil.checkGlError("capture frame"); ShaderUtil.checkGlError("capture frame");
int[] pixelBuffer = new int[bitmapSize]; int[] data = new int[imageSize];
byteBuffer.asIntBuffer().get(pixelBuffer); byteBuffer.asIntBuffer().get(data);
for (int i = 0; i < bitmapSize; i++) { imageCaptureListener.onImageCaptured(texWidth, texHeight, data);
// Swap R and B channels.
pixelBuffer[i] =
(pixelBuffer[i] & 0xff00ff00)
| ((pixelBuffer[i] & 0x000000ff) << 16)
| ((pixelBuffer[i] & 0x00ff0000) >> 16);
}
// Send bitmap.
Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888);
bitmap.setPixels(
pixelBuffer,
/* offset= */ bitmapSize - texWidth,
/* stride= */ -texWidth,
/* x= */ 0,
/* y= */ 0,
texWidth,
texHeight);
bitmapCaptureListener.onBitmapCaptured(bitmap);
} }
GLES20.glBindTexture(textureTarget, 0); GLES20.glBindTexture(textureTarget, 0);
@ -294,6 +275,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
float textureBottom = (1.0f - scaleHeight) * alignmentVertical; float textureBottom = (1.0f - scaleHeight) * alignmentVertical;
float textureTop = textureBottom + scaleHeight; float textureTop = textureBottom + scaleHeight;
Pair<Float, Float> currentZoomLocation = this.zoomLocation;
float zoomLocationX = currentZoomLocation.first;
float zoomLocationY = currentZoomLocation.second;
textureLeft = (textureLeft - 0.5f) / zoomFactor + zoomLocationX;
textureRight = (textureRight - 0.5f) / zoomFactor + zoomLocationX;
textureBottom = (textureBottom - 0.5f) / zoomFactor + zoomLocationY;
textureTop = (textureTop - 0.5f) / zoomFactor + zoomLocationY;
return new float[] {textureLeft, textureRight, textureBottom, textureTop}; return new float[] {textureLeft, textureRight, textureBottom, textureTop};
} }
@ -380,6 +370,22 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
this.scale = scale; this.scale = scale;
} }
/** Zoom factor applied to the frame, must not be 0. */
public void setZoomFactor(float zoomFactor) {
if (zoomFactor == 0.f) {
return;
}
this.zoomFactor = zoomFactor;
}
/**
* Location where to apply the zooming of the frame to. Default is 0.5, 0.5 (scaling is applied to
* the center).
*/
public void setZoomLocation(float zoomLocationX, float zoomLocationY) {
this.zoomLocation = new Pair<>(zoomLocationX, zoomLocationY);
}
private boolean isExternalTexture() { private boolean isExternalTexture() {
return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES; return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
} }

10
mediapipe/java/com/google/mediapipe/framework/PacketGetter.java
View File

@ -128,6 +128,16 @@ public final class PacketGetter {
return ProtoUtil.unpack(result, defaultInstance); return ProtoUtil.unpack(result, defaultInstance);
} }
public static <T extends MessageLite> T getProto(final Packet packet, Parser<T> messageParser) {
SerializedMessage result = new SerializedMessage();
nativeGetProto(packet.getNativeHandle(), result);
try {
return messageParser.parseFrom(result.value);
} catch (InvalidProtocolBufferException e) {
throw new IllegalArgumentException(e);
}
}
/** /**
* @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds. * @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds.
*/ */

Some files were not shown because too many files have changed in this diff Show More