cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed function names in MPPImage Utils

Browse Source
This commit is contained in:
Prianka Liz Kariat 2023-12-11 19:40:13 +05:30
commit 04f826e9d3
381 changed files with 26561 additions and 1515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bazelrc
View File

@ -98,6 +98,9 @@ build:darwin_arm64 --apple_platform_type=macos
build:darwin_arm64 --macos_minimum_os=10.16
build:darwin_arm64 --cpu=darwin_arm64
# Turn off maximum stdout size
build --experimental_ui_max_stdouterr_bytes=-1
# This bazelrc file is meant to be written by a setup script.
try-import %workspace%/.configure.bazelrc

3
WORKSPACE
View File

@ -513,6 +513,9 @@ http_archive(
"@//third_party:org_tensorflow_system_python.diff",
# Diff is generated with a script, don't update it manually.
"@//third_party:org_tensorflow_custom_ops.diff",
# Works around Bazel issue with objc_library.
# See https://github.com/bazelbuild/bazel/issues/19912
"@//third_party:org_tensorflow_objc_build_fixes.diff",
],
patch_args = [
"-p1",

342
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj Normal file
View File

@ -0,0 +1,342 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 56;
objects = {
/* Begin PBXBuildFile section */
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; };
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
8566B5562ABABF9A00AAB22A /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
8566B54F2ABABF9A00AAB22A = {
isa = PBXGroup;
children = (
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
8566B55A2ABABF9A00AAB22A /* Products */,
);
sourceTree = "<group>";
};
8566B55A2ABABF9A00AAB22A /* Products */ = {
isa = PBXGroup;
children = (
8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */,
);
name = Products;
sourceTree = "<group>";
};
8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
isa = PBXGroup;
children = (
8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */,
);
path = MediaPipeTasksDocGen;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXHeadersBuildPhase section */
8566B5542ABABF9A00AAB22A /* Headers */ = {
isa = PBXHeadersBuildPhase;
buildActionMask = 2147483647;
files = (
8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXHeadersBuildPhase section */
/* Begin PBXNativeTarget section */
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
isa = PBXNativeTarget;
buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */;
buildPhases = (
8566B5542ABABF9A00AAB22A /* Headers */,
8566B5552ABABF9A00AAB22A /* Sources */,
8566B5562ABABF9A00AAB22A /* Frameworks */,
8566B5572ABABF9A00AAB22A /* Resources */,
);
buildRules = (
);
dependencies = (
);
name = MediaPipeTasksDocGen;
productName = MediaPipeTasksDocGen;
productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */;
productType = "com.apple.product-type.framework";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
8566B5502ABABF9A00AAB22A /* Project object */ = {
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = 1;
LastUpgradeCheck = 1430;
TargetAttributes = {
8566B5582ABABF9A00AAB22A = {
CreatedOnToolsVersion = 14.3.1;
};
};
};
buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */;
compatibilityVersion = "Xcode 14.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = 8566B54F2ABABF9A00AAB22A;
productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
8566B5572ABABF9A00AAB22A /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
8566B5552ABABF9A00AAB22A /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin XCBuildConfiguration section */
8566B55E2ABABF9A00AAB22A /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
VERSIONING_SYSTEM = "apple-generic";
VERSION_INFO_PREFIX = "";
};
name = Debug;
};
8566B55F2ABABF9A00AAB22A /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
CURRENT_PROJECT_VERSION = 1;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
GCC_C_LANGUAGE_STANDARD = gnu11;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 16.4;
MTL_ENABLE_DEBUG_INFO = NO;
MTL_FAST_MATH = YES;
SDKROOT = iphoneos;
SWIFT_COMPILATION_MODE = wholemodule;
SWIFT_OPTIMIZATION_LEVEL = "-O";
VALIDATE_PRODUCT = YES;
VERSIONING_SYSTEM = "apple-generic";
VERSION_INFO_PREFIX = "";
};
name = Release;
};
8566B5612ABABF9A00AAB22A /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEFINES_MODULE = YES;
DYLIB_COMPATIBILITY_VERSION = 1;
DYLIB_CURRENT_VERSION = 1;
DYLIB_INSTALL_NAME_BASE = "@rpath";
ENABLE_MODULE_VERIFIER = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
8566B5622ABABF9A00AAB22A /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEFINES_MODULE = YES;
DYLIB_COMPATIBILITY_VERSION = 1;
DYLIB_CURRENT_VERSION = 1;
DYLIB_INSTALL_NAME_BASE = "@rpath";
ENABLE_MODULE_VERIFIER = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
MARKETING_VERSION = 1.0;
MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = {
isa = XCConfigurationList;
buildConfigurations = (
8566B55E2ABABF9A00AAB22A /* Debug */,
8566B55F2ABABF9A00AAB22A /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = {
isa = XCConfigurationList;
buildConfigurations = (
8566B5612ABABF9A00AAB22A /* Debug */,
8566B5622ABABF9A00AAB22A /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
};
rootObject = 8566B5502ABABF9A00AAB22A /* Project object */;
}

7
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata generated Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:">
</FileRef>
</Workspace>

8
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>

BIN
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcuserdata/macd.xcuserdatad/UserInterfaceState.xcuserstate generated Normal file
View File

Binary file not shown.

14
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/xcuserdata/macd.xcuserdatad/xcschemes/xcschememanagement.plist Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>MediaPipeTasksDocGen.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>

17
docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h Normal file
View File

@ -0,0 +1,17 @@
//
// MediaPipeTasksDocGen.h
// MediaPipeTasksDocGen
//
// Created by Mark McDonald on 20/9/2023.
//
#import <Foundation/Foundation.h>
//! Project version number for MediaPipeTasksDocGen.
FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber;
//! Project version string for MediaPipeTasksDocGen.
FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[];
// In this header, you should import all the public headers of your framework using statements like
// #import <MediaPipeTasksDocGen/PublicHeader.h>

11
docs/MediaPipeTasksDocGen/Podfile Normal file
View File

@ -0,0 +1,11 @@
# Uncomment the next line to define a global platform for your project
platform :ios, '15.0'
target 'MediaPipeTasksDocGen' do
# Comment the next line if you don't want to use dynamic frameworks
use_frameworks!
# Pods for MediaPipeTasksDocGen
pod 'MediaPipeTasksText'
pod 'MediaPipeTasksVision'
end

9
docs/MediaPipeTasksDocGen/README.md Normal file
View File

@ -0,0 +1,9 @@
# MediaPipeTasksDocGen
This empty project is used to generate reference documentation for the
ObjectiveC and Swift libraries.
Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published
to [the developer site](https://developers.google.com/mediapipe/solutions/).
To bump the API version used, edit [`Podfile`](./Podfile).

2
mediapipe/calculators/audio/spectrogram_calculator.proto
View File

@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the intial input
// the cumulative timestamping, which is inferred from the initial input
// timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 8 [default = false];
}

2
mediapipe/calculators/audio/time_series_framer_calculator.proto
View File

@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the intial input
// the cumulative timestamping, which is inferred from the initial input
// timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 6 [default = false];
}

2
mediapipe/calculators/core/BUILD
View File

@ -727,6 +727,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
],
alwayslink = 1,
)
@ -742,6 +743,7 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"//mediapipe/util:packet_test_util",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],

2
mediapipe/calculators/core/packet_sequencer_calculator_test.cc
View File

@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
}
// Shows how control packets recieve timestamps before and after frame packets
// Shows how control packets receive timestamps before and after frame packets
// have arrived.
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
CalculatorGraphConfig::Node node_config = BuildNodeConfig();

35
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
View File

@ -17,6 +17,7 @@
#include <set>
#include <string>
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
@ -32,6 +33,7 @@ namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK";
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
@ -43,6 +45,7 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset());
res->emplace(kTagAtFirstTick, Timestamp::Unset());
res->emplace(kTagAtTimestamp, Timestamp::Unset());
return res;
}();
@ -59,8 +62,8 @@ std::string GetOutputTag(const CC& cc) {
// timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.)
//
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
// and corresponding timestamps are Timestamp::PreStream(),
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
// input, and timestamp received from a side input.
//
@ -96,6 +99,7 @@ class SidePacketToStreamCalculator : public CalculatorBase {
private:
bool is_tick_processing_ = false;
bool close_on_first_tick_ = false;
std::string output_tag_;
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
@ -103,13 +107,16 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s).";
RET_CHECK(
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
<< "Either both of TICK and AT_TICK should be used or none of them.";
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
"AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
"specify output stream(s).";
const bool has_tick_output =
cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
RET_CHECK((has_tick_output && has_tick_input) ||
(!has_tick_output && !has_tick_input))
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them.";
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
@ -148,11 +155,17 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
if (output_tag_ == kTagAtFirstTick) {
close_on_first_tick_ = true;
}
return absl::OkStatus();
}
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
if (is_tick_processing_) {
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
return absl::OkStatus();
}
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
@ -160,6 +173,9 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
if (close_on_first_tick_) {
cc->Outputs().Get(output_tag_, i).Close();
}
}
return absl::OkStatus();
@ -170,6 +186,7 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
if (!cc->Outputs().HasTag(kTagAtTick) &&
!cc->Outputs().HasTag(kTagAtFirstTick) &&
!cc->Outputs().HasTag(kTagAtTimestamp)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {

166
mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc
View File

@ -27,13 +27,17 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
#include "mediapipe/util/packet_test_util.h"
namespace mediapipe {
namespace {
using testing::HasSubstr;
using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::HasSubstr;
using ::testing::IsEmpty;
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -52,10 +56,35 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both of TICK and AT_TICK should be used or none of them."));
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
TEST(SidePacketToStreamCalculator,
WrongConfigWithMissingTickForFirstTickProcessing) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -76,7 +105,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
"or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -92,14 +121,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
EXPECT_THAT(status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"allowed and required to specify output stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -117,14 +145,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
EXPECT_THAT(status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"allowed and required to specify output stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -146,7 +173,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
"Same number of input side packets and output streams is required."));
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -248,7 +275,50 @@ TEST(SidePacketToStreamCalculator, AtTick) {
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
TEST(SidePacketToStreamCalculator, AtFirstTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value))));
output_packets.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -302,6 +372,62 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_FIRST_TICK:0:packet0"
output_stream: "AT_FIRST_TICK:1:packet1"
}
)pb");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets0,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value0))));
EXPECT_THAT(output_packets1,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value1))));
output_packets0.clear();
output_packets1.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets0, IsEmpty());
EXPECT_THAT(output_packets1, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTimestamp) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
@ -334,7 +460,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
}
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(

2
mediapipe/calculators/core/value_or_default_calculator_test.cc
View File

@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
}
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
// Check that when we provide the inputs not on time - we don't get them.
ValueOrDefaultRunner runner;
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};

2
mediapipe/calculators/image/affine_transformation_runner_opencv.cc
View File

@ -59,7 +59,7 @@ class OpenCvRunner
const ImageFrame& input, const std::array<float, 16>& matrix,
const AffineTransformation::Size& size,
AffineTransformation::BorderMode border_mode) override {
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
// OpenCV warpAffine works in absolute coordinates, so the transform (which
// accepts and produces relative coordinates) should be adjusted to first
// normalize coordinates and then scale them.
// clang-format off

2
mediapipe/calculators/image/image_clone_calculator.cc
View File

@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
}
#else
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true));
cc, /*request_gpu_as_optional=*/true));
#endif // MEDIAPIPE_DISABLE_GPU
return absl::OkStatus();
}

2
mediapipe/calculators/image/image_cropping_calculator.proto
View File

@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
}
// Output texture buffer dimensions. The values defined in the options will be
// overriden by the WIDTH and HEIGHT input streams if they exist.
// overridden by the WIDTH and HEIGHT input streams if they exist.
optional int32 width = 1;
optional int32 height = 2;

6
mediapipe/calculators/image/image_file_properties_calculator.cc
View File

@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
return focal_length_pixels;
}
absl::StatusOr<ImageFileProperties> GetImageFileProperites(
absl::StatusOr<ImageFileProperties> GetImageFileProperties(
const std::string& image_bytes) {
easyexif::EXIFInfo result;
int code = result.parseFrom(image_bytes);
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
if (cc->InputSidePackets().NumEntries() == 1) {
const std::string& image_bytes =
cc->InputSidePackets().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
read_properties_ = true;
}
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
return absl::OkStatus();
}
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
read_properties_ = true;
}
if (read_properties_) {

2
mediapipe/calculators/image/segmentation_smoothing_calculator.cc
View File

@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true));
cc, /*request_gpu_as_optional=*/true));
#endif // !MEDIAPIPE_DISABLE_GPU
return absl::OkStatus();

2
mediapipe/calculators/image/warp_affine_calculator.cc
View File

@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true));
cc, /*request_gpu_as_optional=*/true));
}
return absl::OkStatus();
}

2
mediapipe/calculators/image/warp_affine_calculator_test.cc
View File

@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
.IgnoreError();
mediapipe::GetRotatedSubRectToRectTransformMatrix(
roi_absolute, input.cols, input.rows,
/*flip_horizontaly=*/false, &transform_mat);
/*flip_horizontally=*/false, &transform_mat);
return transform_mat;
}

2
mediapipe/calculators/image/yuv_to_image_calculator.cc
View File

@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
// YV21) format (as per the `fourcc()` property). This covers the most commonly
// used YUV image formats used on mobile devices. Other formats are not
// supported and wil result in an `InvalidArgumentError`.
// supported and will result in an `InvalidArgumentError`.
class YUVToImageCalculator : public Node {
public:
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};

131
mediapipe/calculators/tensor/BUILD
View File

@ -657,6 +657,7 @@ cc_library(
}),
deps = [
":tensor_converter_calculator_cc_proto",
":tensor_converter_cpu",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:image_frame",
@ -665,6 +666,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
@ -674,10 +676,17 @@ cc_library(
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
"//conditions:default": [
"tensor_converter_calculator_gpu_deps",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
}) + select({
"//mediapipe:apple": [
"//third_party/apple_frameworks:MetalKit",
@ -687,6 +696,35 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "tensor_converter_cpu",
srcs = ["tensor_converter_cpu.cc"],
hdrs = ["tensor_converter_cpu.h"],
deps = [
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensor_converter_cpu_test",
srcs = ["tensor_converter_cpu_test.cc"],
deps = [
":tensor_converter_cpu",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"//mediapipe/util:image_test_utils",
],
)
cc_library(
name = "tensor_converter_calculator_gpu_deps",
visibility = ["//visibility:private"],
@ -1414,6 +1452,8 @@ cc_library(
}),
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
@ -1421,9 +1461,11 @@ cc_library(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
@ -1434,6 +1476,7 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:shader_util",
],
}) + selects.with_or({
@ -1453,19 +1496,96 @@ cc_library(
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/port:opencv_imgproc",
":tensors_to_segmentation_converter_opencv",
],
}),
alwayslink = 1,
)
cc_library(
name = "tensors_to_segmentation_utils",
srcs = ["tensors_to_segmentation_utils.cc"],
hdrs = ["tensors_to_segmentation_utils.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensors_to_segmentation_utils_test",
srcs = ["tensors_to_segmentation_utils_test.cc"],
deps = [
":tensors_to_segmentation_utils",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter",
hdrs = ["tensors_to_segmentation_converter.h"],
deps = [
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter_opencv",
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_calculator_test_utils",
testonly = 1,
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "tensors_to_segmentation_calculator_test_utils_test",
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework/port:gtest_main",
],
)
cc_test(
name = "tensors_to_segmentation_calculator_test",
srcs = ["tensors_to_segmentation_calculator_test.cc"],
deps = [
":tensors_to_segmentation_calculator",
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
@ -1476,11 +1596,6 @@ cc_test(
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],
)

4
mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
View File

@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
// Non-streaming mode: when "stream_mode" is set to false in the calculator
// options, the calculators treats the packets in the input audio stream as
// a batch of unrelated audio buffers. In each Process() call, the input
// buffer will be frist resampled, and framed as fixed-sized, possibly
// buffer will be first resampled, and framed as fixed-sized, possibly
// overlapping tensors. The last tensor produced by a Process() invocation
// will be zero-padding if the remaining samples are insufficient. As the
// calculator treats the input packets as unrelated, all samples will be
@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
public:
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
// TODO: Removes this optional input stream when the "AUDIO" stream
// uses the new mediapipe audio data containers that carry audio metatdata,
// uses the new mediapipe audio data containers that carry audio metadata,
// such as sample rate.
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};

2
mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
View File

@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
// will be converted into tensors.
optional double target_sample_rate = 4;
// Whether to treat the input audio stream as a continous stream or a batch
// Whether to treat the input audio stream as a continuous stream or a batch
// of unrelated audio buffers.
optional bool stream_mode = 5 [default = true];

4
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
View File

@ -82,7 +82,7 @@ namespace api2 {
//
// Outputs:
// TENSORS - std::vector<Tensor>
// Vector containing a single Tensor populated with an extrated RGB image.
// Vector containing a single Tensor populated with an extracted RGB image.
// MATRIX - std::array<float, 16> @Optional
// An std::array<float, 16> representing a 4x4 row-major-order matrix that
// maps a point on the input image to a point on the output tensor, and
@ -212,7 +212,7 @@ class ImageToTensorCalculator : public Node {
std::array<float, 16> matrix;
GetRotatedSubRectToRectTransformMatrix(
roi, image->width(), image->height(),
/*flip_horizontaly=*/false, &matrix);
/*flip_horizontally=*/false, &matrix);
kOutMatrix(cc).Send(std::move(matrix));
}

2
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
View File

@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) {
return ImageFormat::GRAY8;
}
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
}
Packet MakeImageFramePacket(cv::Mat input) {

8
mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc
View File

@ -57,7 +57,7 @@ class SubRectExtractorGl {
absl::Status ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta,
bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size,
tflite::gpu::gl::CommandQueue* command_queue,
tflite::gpu::gl::GlBuffer* destination);
@ -154,13 +154,13 @@ void main() {
absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
const tflite::gpu::gl::GlTexture& texture,
const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
bool flip_horizontaly, float alpha, float beta,
bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size,
tflite::gpu::gl::CommandQueue* command_queue,
tflite::gpu::gl::GlBuffer* destination) {
std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
texture_size.h, flip_horizontaly,
texture_size.h, flip_horizontally,
&transform_mat);
MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter {
input_texture,
tflite::gpu::HW(source_texture.height(), source_texture.width()),
roi,
/*flip_horizontaly=*/false, transform.scale, transform.offset,
/*flip_horizontally=*/false, transform.scale, transform.offset,
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
command_queue_.get(), &output));

10
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc
View File

@ -199,7 +199,7 @@ class GlProcessor : public ImageToTensorConverter {
range_min, range_max));
auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
/*flip_horizontaly=*/false,
/*flip_horizontally=*/false,
transform.scale, transform.offset,
output_shape, &tensor_view));
return absl::OkStatus();
@ -210,7 +210,7 @@ class GlProcessor : public ImageToTensorConverter {
absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta,
bool flip_horizontally, float alpha, float beta,
const Tensor::Shape& output_shape,
Tensor::OpenGlTexture2dView* output) {
const int output_height = output_shape.dims[1];
@ -263,13 +263,13 @@ class GlProcessor : public ImageToTensorConverter {
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
GetTransposedRotatedSubRectToRectTransformMatrix(
sub_rect, texture.width(), texture.height(), flip_horizontaly,
sub_rect, texture.width(), texture.height(), flip_horizontally,
&transform_mat);
glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
} else {
GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
texture.height(), flip_horizontaly,
&transform_mat);
texture.height(),
flip_horizontally, &transform_mat);
glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
}

10
mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc
View File

@ -179,13 +179,13 @@ class SubRectExtractorMetal {
}
absl::Status Execute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect, bool flip_horizontaly,
const RotatedRect& sub_rect, bool flip_horizontally,
float alpha, float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLBuffer> destination) {
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
beta, destination_size, command_buffer,
output_texture);
}
@ -211,7 +211,7 @@ class SubRectExtractorMetal {
absl::Status InternalExecute(id<MTLTexture> input_texture,
const RotatedRect& sub_rect,
bool flip_horizontaly, float alpha, float beta,
bool flip_horizontally, float alpha, float beta,
const tflite::gpu::HW& destination_size,
id<MTLCommandBuffer> command_buffer,
id<MTLTexture> output_texture) {
@ -223,7 +223,7 @@ class SubRectExtractorMetal {
std::array<float, 16> transform_mat;
GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
input_texture.height,
flip_horizontaly, &transform_mat);
flip_horizontally, &transform_mat);
id<MTLBuffer> transform_mat_buffer =
[device_ newBufferWithBytes:&transform_mat
length:sizeof(transform_mat)
@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
MtlBufferView::GetWriteView(output_tensor, command_buffer);
MP_RETURN_IF_ERROR(extractor_->Execute(
texture, roi,
/*flip_horizontaly=*/false, transform.scale, transform.offset,
/*flip_horizontally=*/false, transform.scale, transform.offset,
tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
command_buffer, buffer_view.buffer()));
[command_buffer commit];

8
mediapipe/calculators/tensor/image_to_tensor_utils.cc
View File

@ -92,7 +92,7 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
int rect_width, int rect_height,
bool flip_horizontaly,
bool flip_horizontally,
std::array<float, 16>* matrix_ptr) {
std::array<float, 16>& matrix = *matrix_ptr;
// The resulting matrix is multiplication of below commented out matrices:
@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
// {0.0f, 0.0f, a, 0.0f}
// {0.0f, 0.0f, 0.0f, 1.0f}
const float flip = flip_horizontaly ? -1 : 1;
const float flip = flip_horizontally ? -1 : 1;
// Matrix for optional horizontal flip around middle of output image.
// { fl , 0.0f, 0.0f, 0.0f}
// { 0.0f, 1.0f, 0.0f, 0.0f}
@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
void GetTransposedRotatedSubRectToRectTransformMatrix(
const RotatedRect& sub_rect, int rect_width, int rect_height,
bool flip_horizontaly, std::array<float, 16>* matrix_ptr) {
bool flip_horizontally, std::array<float, 16>* matrix_ptr) {
std::array<float, 16>& matrix = *matrix_ptr;
// See comments in GetRotatedSubRectToRectTransformMatrix for detailed
// calculations.
const float a = sub_rect.width;
const float b = sub_rect.height;
const float flip = flip_horizontaly ? -1 : 1;
const float flip = flip_horizontally ? -1 : 1;
const float c = std::cos(sub_rect.rotation);
const float d = std::sin(sub_rect.rotation);
const float e = sub_rect.center_x;

10
mediapipe/calculators/tensor/image_to_tensor_utils.h
View File

@ -74,7 +74,7 @@ absl::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
// Represents a transformation of value which involves scaling and offsetting.
// To apply transformation:
// ValueTransformation transform = ...
// float transformed_value = transform.scale * value + transfrom.offset;
// float transformed_value = transform.scale * value + transform.offset;
struct ValueTransformation {
float scale;
float offset;
@ -99,11 +99,11 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
// @sub_rect - rotated sub rect in absolute coordinates
// @rect_width - rect width
// @rect_height - rect height
// @flip_horizontaly - we need to flip the output buffer.
// @flip_horizontally - we need to flip the output buffer.
// @matrix - 4x4 matrix (array of 16 elements) to populate
void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
int rect_width, int rect_height,
bool flip_horizontaly,
bool flip_horizontally,
std::array<float, 16>* matrix);
// Returns the transpose of the matrix found with
@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
// @sub_rect - rotated sub rect in absolute coordinates
// @rect_width - rect width
// @rect_height - rect height
// @flip_horizontaly - we need to flip the output buffer.
// @flip_horizontally - we need to flip the output buffer.
// @matrix - 4x4 matrix (array of 16 elements) to populate
void GetTransposedRotatedSubRectToRectTransformMatrix(
const RotatedRect& sub_rect, int rect_width, int rect_height,
bool flip_horizontaly, std::array<float, 16>* matrix);
bool flip_horizontally, std::array<float, 16>* matrix);
// Validates the output dimensions set in the option proto. The input option
// proto is expected to have to following fields:

125
mediapipe/calculators/tensor/tensor_converter_calculator.cc
View File

@ -14,6 +14,7 @@
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
#include "absl/log/absl_check.h"
@ -21,17 +22,22 @@
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
#include "absl/types/optional.h"
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer.h"
#if MEDIAPIPE_METAL_ENABLED
#import <CoreVideo/CoreVideo.h>
@ -94,16 +100,13 @@ absl::StatusOr<bool> ShouldFlipVertically(
}
}
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kMatrixTag[] = "MATRIX";
constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
} // namespace
namespace mediapipe {
@ -156,10 +159,6 @@ class TensorConverterCalculator : public CalculatorBase {
private:
absl::Status InitGpu(CalculatorContext* cc);
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame,
bool flip_vertically, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
absl::Status ProcessCPU(CalculatorContext* cc);
absl::Status ProcessGPU(CalculatorContext* cc);
@ -279,46 +278,21 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
}
const auto& image_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels_);
const mediapipe::ImageFormat::Format format = image_frame.Format();
if (!(format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
output_tensors->emplace_back(
Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensors->back().GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
MP_ASSIGN_OR_RETURN(Tensor output,
ConvertImageFrameToTensorOnCpu(
image_frame,
output_range_.has_value() ? output_range_.value()
: kDefaultOutputRange,
flip_vertically_, max_num_channels_));
output_tensors->emplace_back(std::move(output));
} else if (cc->Inputs().HasTag(kMatrixTag)) {
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
return absl::OkStatus();
}
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
MP_ASSIGN_OR_RETURN(Tensor output,
ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
output_tensors->emplace_back(std::move(output));
} else {
return absl::OkStatus();
}
@ -669,67 +643,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
return absl::OkStatus();
}
template <class T>
absl::Status TensorConverterCalculator::NormalizeImage(
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels_);
const int channels_ignored = channels - channels_preserved;
if (output_range_.has_value()) {
// If the output float range is set and we are not using custom
// normalization, normalize the pixel values from [0, 255] to the specified
// output range.
RET_CHECK_NE(output_range_->first, output_range_->second);
const float scale = (output_range_->second - output_range_->first) / 255.0f;
const float bias = output_range_->first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
} else {
// [0,1], scale only (bias == 0)
// Verified that there are no precision issues with 1.0f / 255.0f expression
const float scale = 1.0f / 255.0f;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale;
}
image_ptr += channels_ignored;
}
}
}
return absl::OkStatus();
}
absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
float* tensor_ptr) {
if (row_major_matrix_) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
} // namespace mediapipe

2
mediapipe/calculators/tensor/tensor_converter_calculator.proto
View File

@ -32,7 +32,7 @@ message TensorConverterCalculatorOptions {
// Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overriden, and
// values take effect, the |zero_center| setting above will be overridden, and
// the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false];

55
mediapipe/calculators/tensor/tensor_converter_calculator_test.cc
View File

@ -321,6 +321,61 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
}
}
TEST_F(TensorConverterCalculatorTest,
ShouldConvertImageWithDefaultOutputRange) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
zero_center: false
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
mat.at<uint8_t>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1);
// Get and process results.
const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1);
const Tensor* tensor = &tensor_vec[0];
// Calculate the expected normalized value:
float expected_value = 200.0 / 255.0;
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
auto view = tensor->GetCpuReadView();
float actual_value = *view.buffer<float>();
EXPECT_FLOAT_EQ(actual_value, expected_value);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(TensorConverterCalculatorTest, FlipVertically) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =

145
mediapipe/calculators/tensor/tensor_converter_cpu.cc Normal file
View File

@ -0,0 +1,145 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <algorithm>
#include <cstdint>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const int channels_ignored = channels - channels_preserved;
RET_CHECK_NE(output_range.first, output_range.second);
const float scale = (output_range.second - output_range.first) / 255.0f;
const float bias = output_range.first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
return absl::OkStatus();
}
} // namespace
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<float>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr) {
if (is_row_major_matrix) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const mediapipe::ImageFormat::Format format = image_frame.Format();
if (!(format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensor.GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
return output_tensor;
}
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix) {
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(
CopyMatrixToTensor(matrix, row_major_matrix,
output_tensor.GetCpuWriteView().buffer<float>()));
return output_tensor;
}
} // namespace mediapipe

61
mediapipe/calculators/tensor/tensor_converter_cpu.h Normal file
View File

@ -0,0 +1,61 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
// Converts an ImageFrame to a vector of Tensors.
// @flip_vertically enables to flip the image during conversion.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels);
// Converts a Matrix to a vector of Tensors.
// @row_major_matrix defines the ordering in the input matrix.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix);
// For testing only below.
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_

175
mediapipe/calculators/tensor/tensor_converter_cpu_test.cc Normal file
View File

@ -0,0 +1,175 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <cstdint>
#include <utility>
#include <vector>
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/util/image_test_utils.h"
namespace mediapipe {
namespace {
Matrix CreateTestMatrix(int num_rows, int num_columns) {
Matrix matrix(num_rows, num_columns);
for (int r = 0; r < num_rows; ++r) {
for (int c = 0; c < num_columns; ++c) {
matrix(r, c) = r * num_columns + c;
}
}
return matrix;
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i / test_matrix.cols();
const int column = i % test_matrix.cols();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i % test_matrix.rows();
const int column = i / test_matrix.rows();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(
tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
const auto range = std::make_pair(2.0f, 3.0f);
MP_EXPECT_OK(
NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
/*num_tensor_channels=*/1, tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
255.0f * (range.second - range.first) +
range.first);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int x = i % grey8_image_frame.Width();
const int y = i / grey8_image_frame.Width();
const int flipped_y = grey8_image_frame.Height() - y - 1;
const int index = flipped_y * grey8_image_frame.Width() + x;
EXPECT_FLOAT_EQ(
tensor_data[index],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
auto float_image_frame =
CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
float_image_frame.Width() * float_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
float_image_frame.PixelData())[i] /
255.0f);
}
}
TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
grey8_image_frame, {0.0f, 1.0f},
/*flip_vertically=*/false,
/*max_num_channels=*/1));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
++i) {
EXPECT_FLOAT_EQ(
tensor_ptr[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
}
}
TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
MP_ASSERT_OK_AND_ASSIGN(
Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
/*row_major_matrix=*/false));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < test_matrix.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
}
}
} // namespace
} // namespace mediapipe

2
mediapipe/calculators/tensor/tensors_to_classification_calculator.proto
View File

@ -34,7 +34,7 @@ message TensorsToClassificationCalculatorOptions {
repeated Entry entries = 1;
}
// Score threshold for perserving the class.
// Score threshold for preserving the class.
optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then
// all labels are used.

60
mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
View File

@ -15,7 +15,6 @@
#include <unordered_map>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h"
#include "absl/types/span.h"
#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
@ -147,7 +146,7 @@ BoxFormat GetBoxFormat(const TensorsToDetectionsCalculatorOptions& options) {
// TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have
// 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints.
// The size of the values must be (num_boxes * num_predicted_values).
// Second tensor is the score tensor. The size of the valuse must be
// Second tensor is the score tensor. The size of the values must be
// (num_boxes * num_classes). It's optional to pass in a third tensor
// for anchors (e.g. for SSD models) depend on the outputs of the
// detection model. The size of anchor tensor must be (num_boxes *
@ -215,7 +214,8 @@ class TensorsToDetectionsCalculator : public Node {
const int* detection_classes,
std::vector<Detection>* output_detections);
Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
float box_xmax, float score, int class_id,
float box_xmax, absl::Span<const float> scores,
absl::Span<const int> class_ids,
bool flip_vertically);
bool IsClassIndexAllowed(int class_index);
@ -223,6 +223,7 @@ class TensorsToDetectionsCalculator : public Node {
int num_boxes_ = 0;
int num_coords_ = 0;
int max_results_ = -1;
int classes_per_detection_ = 1;
BoxFormat box_output_format_ =
mediapipe::TensorsToDetectionsCalculatorOptions::YXHW;
@ -267,7 +268,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract(
if (CanUseGpu()) {
#ifndef MEDIAPIPE_DISABLE_GL_COMPUTE
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true));
cc, /*request_gpu_as_optional=*/true));
#elif MEDIAPIPE_METAL_ENABLED
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@ -484,6 +485,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
auto num_boxes_view = num_boxes_tensor->GetCpuReadView();
auto num_boxes = num_boxes_view.buffer<float>();
num_boxes_ = num_boxes[0];
// The detection model with Detection_PostProcess op may output duplicate
// boxes with different classes, in the following format:
// num_boxes_tensor = [num_boxes]
// detection_classes_tensor = [box_1_class_1, box_1_class_2, ...]
// detection_scores_tensor = [box_1_score_1, box_1_score_2, ... ]
// detection_boxes_tensor = [box_1, box1, ... ]
// Each box repeats classes_per_detection_ times.
// Note Detection_PostProcess op is only supported in CPU.
RET_CHECK_EQ(max_detections % num_boxes_, 0);
classes_per_detection_ = max_detections / num_boxes_;
auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView();
auto detection_boxes = detection_boxes_view.buffer<float>();
@ -493,8 +504,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
auto detection_classes_view = detection_classes_tensor->GetCpuReadView();
auto detection_classes_ptr = detection_classes_view.buffer<float>();
std::vector<int> detection_classes(num_boxes_);
for (int i = 0; i < num_boxes_; ++i) {
std::vector<int> detection_classes(num_boxes_ * classes_per_detection_);
for (int i = 0; i < detection_classes.size(); ++i) {
detection_classes[i] = static_cast<int>(detection_classes_ptr[i]);
}
MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores,
@ -863,24 +874,25 @@ absl::Status TensorsToDetectionsCalculator::DecodeBoxes(
absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
const float* detection_boxes, const float* detection_scores,
const int* detection_classes, std::vector<Detection>* output_detections) {
for (int i = 0; i < num_boxes_; ++i) {
for (int i = 0; i < num_boxes_ * classes_per_detection_;
i += classes_per_detection_) {
if (max_results_ > 0 && output_detections->size() == max_results_) {
break;
}
if (options_.has_min_score_thresh() &&
detection_scores[i] < options_.min_score_thresh()) {
continue;
}
if (!IsClassIndexAllowed(detection_classes[i])) {
continue;
}
const int box_offset = i * num_coords_;
Detection detection = ConvertToDetection(
/*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
/*box_xmin=*/detection_boxes[box_offset + box_indices_[1]],
/*box_ymax=*/detection_boxes[box_offset + box_indices_[2]],
/*box_xmax=*/detection_boxes[box_offset + box_indices_[3]],
detection_scores[i], detection_classes[i], options_.flip_vertically());
absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
options_.flip_vertically());
// if all the scores and classes are filtered out, we skip the empty
// detection.
if (detection.score().empty()) {
continue;
}
const auto& bbox = detection.location_data().relative_bounding_box();
if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
std::isnan(bbox.height())) {
@ -910,11 +922,21 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
}
Detection TensorsToDetectionsCalculator::ConvertToDetection(
float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score,
int class_id, bool flip_vertically) {
float box_ymin, float box_xmin, float box_ymax, float box_xmax,
absl::Span<const float> scores, absl::Span<const int> class_ids,
bool flip_vertically) {
Detection detection;
detection.add_score(score);
detection.add_label_id(class_id);
for (int i = 0; i < scores.size(); ++i) {
if (!IsClassIndexAllowed(class_ids[i])) {
continue;
}
if (options_.has_min_score_thresh() &&
scores[i] < options_.min_score_thresh()) {
continue;
}
detection.add_score(scores[i]);
detection.add_label_id(class_ids[i]);
}
LocationData* location_data = detection.mutable_location_data();
location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);

2
mediapipe/calculators/tensor/tensors_to_detections_calculator.proto
View File

@ -75,7 +75,7 @@ message TensorsToDetectionsCalculatorOptions {
// representation has a bottom-left origin (e.g., in OpenGL).
optional bool flip_vertically = 18 [default = false];
// Score threshold for perserving decoded detections.
// Score threshold for preserving decoded detections.
optional float min_score_thresh = 19;
// The maximum number of the detection results to return. If < 0, all

2
mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
View File

@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
kFlipVertically(cc).IsConnected())) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input width/height for using flipping when outputing "
<< "Must provide input width/height for using flipping when outputting "
"landmarks in absolute coordinates.";
}
return absl::OkStatus();

223
mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
View File

@ -12,32 +12,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "absl/strings/str_format.h"
#include "absl/types/span.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/shader_util.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
@ -62,37 +65,9 @@ namespace {
constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size) { // NOLINT
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
constexpr char kMaskTag[] = "MASK";
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace
namespace mediapipe {
@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
private:
absl::Status LoadOptions(CalculatorContext* cc);
absl::Status InitGpu(CalculatorContext* cc);
absl::Status ProcessGpu(CalculatorContext* cc);
absl::Status ProcessCpu(CalculatorContext* cc);
absl::Status ProcessGpu(CalculatorContext* cc,
const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width,
int output_height);
void GlRender();
bool DoesGpuTextureStartAtBottom() {
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
}
absl::Status InitConverterIfNecessary() {
#if !MEDIAPIPE_DISABLE_OPENCV
template <class T>
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
if (!cpu_converter_) {
MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
}
#else
RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV
::mediapipe::TensorsToSegmentationCalculatorOptions options_;
return absl::OkStatus();
}
mediapipe::TensorsToSegmentationCalculatorOptions options_;
std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
#if !MEDIAPIPE_DISABLE_GPU
mediapipe::GlCalculatorHelper gpu_helper_;
@ -208,7 +192,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract(
if (CanUseGpu()) {
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*requesst_gpu_as_optional=*/true));
cc, /*request_gpu_as_optional=*/true));
#if MEDIAPIPE_METAL_ENABLED
MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
#endif // MEDIAPIPE_METAL_ENABLED
@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
MP_ASSIGN_OR_RETURN(auto hwc,
GetHwcFromDims(input_tensors[0].shape().dims));
int tensor_channels = std::get<2>(hwc);
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
switch (options_.activation()) {
case Options::NONE:
RET_CHECK_EQ(tensor_channels, 1);
@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
}
}
// Get dimensions.
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
if (use_gpu) {
#if !MEDIAPIPE_DISABLE_GPU
if (!gpu_initialized_) {
@ -286,8 +281,11 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
MP_RETURN_IF_ERROR(ProcessGpu(cc));
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
output_height, hwc]() -> absl::Status {
MP_RETURN_IF_ERROR(
ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
return absl::OkStatus();
}));
#else
@ -295,7 +293,13 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
#if !MEDIAPIPE_DISABLE_OPENCV
MP_RETURN_IF_ERROR(ProcessCpu(cc));
// Lazily initialize converter.
MP_RETURN_IF_ERROR(InitConverterIfNecessary());
MP_ASSIGN_OR_RETURN(
std::unique_ptr<Image> output_mask,
cpu_converter_->Convert(input_tensors, output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
cc->InputTimestamp());
#else
RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV
@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
absl::Status TensorsToSegmentationCalculator::ProcessCpu(
CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_OPENCV
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
#endif // !MEDIAPIPE_DISABLE_OPENCV
return absl::OkStatus();
}
#if !MEDIAPIPE_DISABLE_OPENCV
template <class T>
absl::Status TensorsToSegmentationCalculator::ApplyActivation(
cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
#endif // !MEDIAPIPE_DISABLE_OPENCV
// Steps:
// 1. receive tensor
// 2. process segmentation tensor into small mask
// 3. upsample small mask into output mask to be same size as input image
absl::Status TensorsToSegmentationCalculator::ProcessGpu(
CalculatorContext* cc) {
CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width, int output_height) {
#if !MEDIAPIPE_DISABLE_GPU
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask texture.
#if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() {
absl::Status TensorsToSegmentationCalculator::LoadOptions(
CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>();
options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
return absl::OkStatus();
}
@ -826,7 +713,7 @@ void main() {
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
// Shader defines.
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
const std::string output_layer_index =
"\n#define OUTPUT_LAYER_INDEX int(" +
std::to_string(options_.output_layer_index()) + ")";

191
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc
View File

@ -17,10 +17,8 @@
#include <utility>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/log/log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image.h"
@ -30,7 +28,6 @@
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
@ -40,62 +37,17 @@ namespace {
using ::testing::SizeIs;
using ::testing::TestWithParam;
using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
std::string ActivationTypeToString(Options::Activation activation) {
switch (activation) {
case Options::NONE:
return "NONE";
case Options::SIGMOID:
return "SIGMOID";
case Options::SOFTMAX:
return "SOFTMAX";
default:
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
}
using TensorsToSegmentationCalculatorTest =
TestWithParam<test_utils::FormattingTestCase>;
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
Options::Activation activation;
int rows;
int cols;
int channels;
};
using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
// Currently only useable for tests with no output resize.
TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
const FormattingTestCase& test_case = GetParam();
std::vector<float> inputs = test_case.inputs;
std::vector<float> expected_outputs = test_case.expected_outputs;
Options::Activation activation = test_case.activation;
int rows = test_case.rows;
int cols = test_case.cols;
int channels = test_case.channels;
const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
rows_new, cols_new, channels, max_abs_diff] = GetParam();
std::string string_config = absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size"
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $0
}
}
}
)pb",
ActivationTypeToString(activation));
auto graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(string_config);
test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
std::vector<Packet> output_packets;
tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
@ -119,28 +71,34 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0))));
}
// The output size is defined as pair(new_width, new_height).
MP_ASSERT_OK(graph.AddPacketToInputStream(
"size",
mediapipe::Adopt(new std::pair<int, int>(rows, cols)).At(Timestamp(0))));
"size", mediapipe::Adopt(new std::pair<int, int>(cols_new, rows_new))
.At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_packets, SizeIs(1));
const Image& image_as_mask = output_packets[0].Get<Image>();
EXPECT_FALSE(image_as_mask.UsesGpu());
std::shared_ptr<cv::Mat> result_mat = formats::MatView(&image_as_mask);
EXPECT_EQ(result_mat->rows, rows);
EXPECT_EQ(result_mat->cols, cols);
EXPECT_EQ(result_mat->channels(), channels);
EXPECT_EQ(result_mat->rows, rows_new);
EXPECT_EQ(result_mat->cols, cols_new);
EXPECT_EQ(result_mat->channels(), 1);
// Compare the real result with the expected result.
cv::Mat expected_result = cv::Mat(
rows, cols, CV_32FC1, const_cast<float*>(expected_outputs.data()));
cv::Mat expected_result =
cv::Mat(rows_new, cols_new, CV_32FC1,
const_cast<float*>(expected_outputs.data()));
cv::Mat diff;
cv::absdiff(*result_mat, expected_result, diff);
double max_val;
cv::minMaxLoc(diff, nullptr, &max_val);
// Expects the maximum absolute pixel-by-pixel difference is less than 1e-5.
// This delta is for passthorugh accuracy only.
EXPECT_LE(max_val, 1e-5);
// The max allowable diff between output and expected output varies between
// tests.
EXPECT_LE(max_val, max_abs_diff);
MP_ASSERT_OK(graph.CloseInputStream("tensors"));
MP_ASSERT_OK(graph.CloseInputStream("size"));
@ -149,18 +107,97 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
INSTANTIATE_TEST_SUITE_P(
TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
testing::ValuesIn<FormattingTestCase>({
{/*test_name=*/"NoActivationAndNoOutputResize",
/*inputs=*/
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
14.0, 15.0, 16.0},
/*expected_outputs=*/
{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
14.0, 15.0, 16.0},
/*activation=*/Options::NONE,
/*rows=*/4,
/*cols=*/4,
/*channels=*/1},
testing::ValuesIn<test_utils::FormattingTestCase>({
{.test_name = "NoActivationAndNoOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
11.0, 12.0, 13.0, 14.0, 15.0, 16.0},
.activation = Options::NONE,
.rows = 4,
.cols = 4,
.rows_new = 4,
.cols_new = 4,
.channels = 1,
.max_abs_diff = 1e-7},
{.test_name = "OutputResizeOnly",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {1, 1.5, 2.166667, 2.833333, 3.5, 4,
3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8,
7, 7.5, 8.166667, 8.833333, 9.5, 10,
10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2,
13, 13.5, 14.166667, 14.833333, 15.5, 16},
.activation = Options::NONE,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SigmoidActivationWithNoOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {0.731059, 0.880797, 0.952574, 0.982014, 0.993307,
0.997527, 0.999089, 0.999665, 0.999877, 0.999955,
0.999983, 0.999994, 0.999998, 0.999999, 1.0, 1.0},
.activation = Options::SIGMOID,
.rows = 4,
.cols = 4,
.rows_new = 4,
.cols_new = 4,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SigmoidActivationWithOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},
.expected_outputs = {0.731059, 0.805928, 0.89276, 0.940611, 0.967294,
0.982014, 0.914633, 0.93857, 0.966279, 0.981363,
0.989752, 0.994369, 0.996592, 0.997666, 0.998873,
0.999404, 0.999683, 0.999829, 0.999913, 0.99994,
0.999971, 0.999985, 0.999992, 0.999996, 0.999998,
0.999998, 0.999999, 1.0, 1.0, 1.0},
.activation = Options::SIGMOID,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 1,
.max_abs_diff = 1e-6},
{.test_name = "SoftmaxActivationWithNoOutputResize",
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
.expected_outputs = {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574,
0.000911051, 0.952574, 0.924142, 0.731059,
0.731059, 0.24974, 0.937027, 0.689974, 0.990048,
0.0060598, 0.28905},
.activation = Options::SOFTMAX,
.rows = 4,
.cols = 4,
.rows_new = 4,
.cols_new = 4,
.channels = 2,
.max_abs_diff = 1e-6},
{.test_name = "SoftmaxActivationWithOutputResize",
.inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5,
7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5,
19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
.expected_outputs = {0.731059, 0.425131, 0.246135, 0.753865, 0.445892,
0.0109869, 0.886119, 0.461259, 0.185506, 0.781934,
0.790618, 0.650195, 0.841816, 0.603901, 0.40518,
0.561962, 0.765871, 0.930584, 0.718733, 0.763744,
0.703402, 0.281989, 0.459635, 0.742634, 0.689974,
0.840011, 0.82605, 0.170058, 0.147555, 0.28905},
.activation = Options::SOFTMAX,
.rows = 4,
.cols = 4,
.rows_new = 5,
.cols_new = 6,
.channels = 2,
.max_abs_diff = 1e-6},
}),
[](const testing::TestParamInfo<
TensorsToSegmentationCalculatorTest::ParamType>& info) {

111
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc Normal file
View File

@ -0,0 +1,111 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <string>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/port/parse_text_proto.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
switch (activation) {
case TensorsToSegmentationCalculatorOptions::NONE:
return "NONE";
case TensorsToSegmentationCalculatorOptions::SIGMOID:
return "SIGMOID";
case TensorsToSegmentationCalculatorOptions::SOFTMAX:
return "SOFTMAX";
}
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array) {
std::vector<unsigned char> result;
result.reserve(array.size());
for (int i = 0; i < array.size(); ++i) {
result.push_back(static_cast<unsigned char>(array[i]));
}
return result;
}
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
std::vector<float> result;
result.reserve(values.size() * 4);
for (const float& value : values) {
result.push_back(value);
result.push_back(0);
result.push_back(0);
result.push_back(value);
}
return result;
}
// For GPU tests, the input tensor needs to be moved to GPU, using
// TensorViewRequestor. After calculation, the output needs to be moved back
// to CPU, using ToImageCalculator. The output is an ImageFrame.
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
std::string pre_process = R"pb(
node {
calculator: "mediapipe.aimatter.TensorViewRequestor"
input_stream: "TENSORS:tensors"
output_stream: "TENSORS:tensors_gpu"
options {
[mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
}
}
)pb";
std::string post_process = R"pb(
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:image_as_mask_gpu"
output_stream: "IMAGE_CPU:image_as_mask"
}
)pb";
return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size" $0
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors$1"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask$2"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $3
gpu_origin: TOP_LEFT
}
}
} $4
)pb",
test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
test_gpu ? post_process : ""));
}
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe

57
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array);
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
int rows = 1;
int cols = 1;
int rows_new = 1;
int cols_new = 1;
int channels = 1;
double max_abs_diff = 1e-7;
};
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_

50
mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/port/gtest.h"
namespace mediapipe::tensors_to_segmentation_utils {
namespace {
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ActivationTypeToStringWorksCorrectly) {
EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ArrayFloatToUnsignedCharWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<unsigned char> expected = {1, 2, 3};
EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
MakeRedAlphaMatrixWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
}
} // namespace
} // namespace mediapipe::tensors_to_segmentation_utils

43
mediapipe/calculators/tensor/tensors_to_segmentation_converter.h Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#include <memory>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
class TensorsToSegmentationConverter {
public:
virtual ~TensorsToSegmentationConverter() = default;
// Converts tensors to image mask.
// Returns a unique pointer containing the converted image.
// @input_tensors contains the tensors needed to be processed.
// @output_width/height describes output dimensions to reshape the output mask
// into.
virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) = 0;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_

157
mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc Normal file
View File

@ -0,0 +1,157 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#include <algorithm>
#include <cmath>
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
class OpenCvProcessor : public TensorsToSegmentationConverter {
public:
absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
options_ = options;
return absl::OkStatus();
}
absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) override;
private:
template <class T>
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
TensorsToSegmentationCalculatorOptions options_;
};
absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) {
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
auto output_mask = std::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
return output_mask;
}
template <class T>
absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
} // namespace
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
auto converter = std::make_unique<OpenCvProcessor>();
MP_RETURN_IF_ERROR(converter->Init(options));
return converter;
}
} // namespace mediapipe

31
mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#include <memory>
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
namespace mediapipe {
// Creates OpenCV tensors-to-segmentation converter.
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(
const mediapipe::TensorsToSegmentationCalculatorOptions& options);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_

52
mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc Normal file
View File

@ -0,0 +1,52 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
int NumGroups(int size, int group_size) {
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace mediapipe

34
mediapipe/calculators/tensor/tensors_to_segmentation_utils.h Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
namespace mediapipe {
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size); // NOLINT
bool CanUseGpu();
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_

63
mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc Normal file
View File

@ -0,0 +1,63 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
using ::testing::HasSubstr;
TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
EXPECT_EQ(NumGroups(13, 4), 4);
EXPECT_EQ(NumGroups(4, 13), 1);
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
std::vector<int> dims_3 = {2, 3, 4};
absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
MP_ASSERT_OK(result_1);
EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
std::vector<int> dims_4 = {1, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
MP_ASSERT_OK(result_2);
EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
std::vector<int> dims_4 = {2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Expected batch to be 1 for BHWC heatmap"));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
std::vector<int> dims_5 = {1, 2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Invalid shape for segmentation tensor"));
}
} // namespace
} // namespace mediapipe

2
mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
View File

@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
// and label and label_id are optional but at least one of them should be set.
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
// prefixed versions of each stream, which allows for multiple image streams to
// be included. However, the default names are suppored by more tools.
// be included. However, the default names are supported by more tools.
//
// Example config:
// node {

4
mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc
View File

@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
// -- 1-D or 2-D Tensor
// Output:
// -- Matrix with the same values as the Tensor
// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape.
// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape.
// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
//
// Example Config
// node: {

2
mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc
View File

@ -15,9 +15,9 @@
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
#include <cstdint>
#include <memory>
#include "absl/base/integral_types.h"
#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"

6
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
View File

@ -111,8 +111,8 @@ class InferenceState {
// input_side_packet.
//
// The input and output streams are TensorFlow tensors labeled by tags. The tags
// for the streams are matched to feeds and fetchs in a TensorFlow session using
// a named_signature.generic_signature in the ModelManifest. The
// for the streams are matched to feeds and fetches in a TensorFlow session
// using a named_signature.generic_signature in the ModelManifest. The
// generic_signature is used as key-value pairs between the MediaPipe tag and
// the TensorFlow tensor. The signature_name in the options proto determines
// which named_signature is used. The keys in the generic_signature must be
@ -128,7 +128,7 @@ class InferenceState {
// addition. Once batch_size inputs have been provided, the batch will be run
// and the output tensors sent out on the output streams with timestamps
// corresponding to the input stream packets. Setting the batch_size to 1
// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
// completely disables batching, but is independent of add_batch_dim_to_tensors.
//
// The TensorFlowInferenceCalculator also support feeding states recurrently for
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the

2
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto
View File

@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
// If the 0th dimension is the batch dimension, then the tensors are
// concatenated on that dimension. If the 0th is a data dimension, then a 0th
// dimension is added before concatenating. If added, the extra dimension is
// removed before outputing the tensor. Examples of each case: If you want
// removed before outputting the tensor. Examples of each case: If you want
// to batch spectra of audio over time for an LSTM, a time-frequency
// representation has a 0th dimension as the batch dimension. If you want to
// batch frames of video that are [width, height, channels], the batch

2
mediapipe/calculators/tflite/testdata/README.md vendored
View File

@ -1,2 +1,2 @@
The model files add.bin, add_quantized.bin
(and corresponding metatada json files) come from tensorflow/lite/testdata/
(and corresponding metadata json files) come from tensorflow/lite/testdata/

2
mediapipe/calculators/tflite/tflite_converter_calculator.cc
View File

@ -95,7 +95,7 @@ struct GPUData {
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
// or MTLBuffer.
//
// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
// This calculator is designed to be used with the TfLiteInferenceCalculator,
// as a pre-processing step for calculator inputs.
//
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],

2
mediapipe/calculators/tflite/tflite_converter_calculator.proto
View File

@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
// Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overriden, and
// values take effect, the |zero_center| setting above will be overridden, and
// the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false];

2
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto
View File

@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
}
// Score threshold for perserving the class.
// Score threshold for preserving the class.
optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then
// all labels are used.

2
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc
View File

@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
// tensors can have 2 or 3 tensors. First tensor is the predicted
// raw boxes/keypoints. The size of the values must be (num_boxes
// * num_predicted_values). Second tensor is the score tensor. The
// size of the valuse must be (num_boxes * num_classes). It's
// size of the values must be (num_boxes * num_classes). It's
// optional to pass in a third tensor for anchors (e.g. for SSD
// models) depend on the outputs of the detection model. The size
// of anchor tensor must be (num_boxes * 4).

2
mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto
View File

@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
// representation has a bottom-left origin (e.g., in OpenGL).
optional bool flip_vertically = 18 [default = false];
// Score threshold for perserving decoded detections.
// Score threshold for preserving decoded detections.
optional float min_score_thresh = 19;
}

2
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
View File

@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input width/height for using flip_vertically option "
"when outputing landmarks in absolute coordinates.";
"when outputting landmarks in absolute coordinates.";
}
flip_horizontally_ =

BIN
mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

3
mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties vendored
View File

@ -1,6 +1,7 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

29
mediapipe/examples/android/solutions/gradlew vendored
View File

@ -83,10 +83,8 @@ done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
@ -133,10 +131,13 @@ location of your Java installation."
fi
else
JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC3045
# shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC3045
# shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
done
fi
# Collect all arguments for the java command;
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
# shell script including quotes and variable substitutions, so put them in
# double quotes to make sure that they get re-expanded; and
# * put everything else in single quotes, so that it's not re-expanded.
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \

2
mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc
View File

@ -55,7 +55,7 @@ absl::Status RunMPPGraph() {
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
RET_CHECK(!input_side_packets.contains(name_and_value[0]));
std::string input_side_packet_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
name_and_value[1], &input_side_packet_contents));

2
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc
View File

@ -56,7 +56,7 @@ absl::Status RunMPPGraph() {
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
RET_CHECK(!input_side_packets.contains(name_and_value[0]));
std::string input_side_packet_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
name_and_value[1], &input_side_packet_contents));

16
mediapipe/framework/api2/builder.h
View File

@ -330,6 +330,14 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
class NodeBase {
public:
NodeBase() = default;
~NodeBase() = default;
NodeBase(NodeBase&&) = default;
NodeBase& operator=(NodeBase&&) = default;
// Explicitly delete copies to improve error messages.
NodeBase(const NodeBase&) = delete;
NodeBase& operator=(const NodeBase&) = delete;
// TODO: right now access to an indexed port is made directly by
// specifying both a tag and an index. It would be better to represent this
// as a two-step lookup, first getting a multi-port, and then accessing one
@ -585,6 +593,14 @@ class PacketGenerator {
class Graph {
public:
Graph() = default;
~Graph() = default;
Graph(Graph&&) = default;
Graph& operator=(Graph&&) = default;
// Explicitly delete copies to improve error messages.
Graph(const Graph&) = delete;
Graph& operator=(const Graph&) = delete;
void SetType(std::string type) { type_ = std::move(type); }
// Creates a node of a specific type. Should be used for calculators whose

9
mediapipe/framework/formats/BUILD
View File

@ -124,6 +124,15 @@ cc_library(
],
)
cc_library(
name = "ahwb_view",
hdrs = ["ahwb_view.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/gpu:gpu_buffer_storage",
],
)
cc_library(
name = "affine_transform",
srcs = ["affine_transform.cc"],

54
mediapipe/framework/formats/ahwb_view.h Normal file
View File

@ -0,0 +1,54 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#include "mediapipe/framework/port.h"
#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
#include <android/hardware_buffer.h>
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
// Intended use cases:
// - Extracting an AHWB for processing in another library after it's produced by
// MediaPipe.
// - Sending AHWBs to compute devices that are able to map the memory for their
// own usage.
// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
// other CPU/GPU uses of AHWBs.
class AhwbView {
public:
explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
// Non-copyable
AhwbView(const AhwbView&) = delete;
AhwbView& operator=(const AhwbView&) = delete;
// Non-movable
AhwbView(AhwbView&&) = delete;
// Only supports synchronous usage. All users of GetHandle must finish
// accessing the buffer before this view object is destroyed to avoid race
// conditions.
// TODO: Support asynchronous usage.
const AHardwareBuffer* GetHandle() const { return handle_; }
private:
const AHardwareBuffer* handle_;
};
namespace internal {
// Makes this class available as a GpuBuffer view.
template <>
class ViewProvider<AhwbView> {
public:
virtual ~ViewProvider() = default;
virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
virtual AhwbView GetWriteView(types<AhwbView>) = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_

7
mediapipe/framework/port.h
View File

@ -50,7 +50,7 @@
// but may or may not still be able to run other OpenGL code.
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
(defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
MEDIAPIPE_USING_SWIFTSHADER)
MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
#define MEDIAPIPE_DISABLE_GL_COMPUTE
#endif
@ -104,4 +104,9 @@
#endif
#endif // MEDIAPIPE_HAS_RTTI
// AHardware buffers are only available since Android API 26.
#if (__ANDROID_API__ >= 26)
#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
#endif
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_

3
mediapipe/framework/tool/BUILD
View File

@ -616,6 +616,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/functional:bind_front",
"@com_google_absl//absl/strings",
],
)
@ -856,6 +857,7 @@ cc_library(
mediapipe_cc_test(
name = "switch_demux_calculator_test",
srcs = ["switch_demux_calculator_test.cc"],
requires_full_emulation = False,
deps = [
":container_util",
":switch_demux_calculator",
@ -891,6 +893,7 @@ cc_library(
mediapipe_cc_test(
name = "switch_mux_calculator_test",
srcs = ["switch_mux_calculator_test.cc"],
requires_full_emulation = False,
deps = [
":container_util",
":switch_mux_calculator",

1
mediapipe/framework/tool/sink_test.cc
View File

@ -17,6 +17,7 @@
#include <memory>
#include <vector>
#include "absl/functional/bind_front.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"

4
mediapipe/framework/tool/validate_name.cc
View File

@ -134,7 +134,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
RET_CHECK(name);
absl::Status tag_status = absl::OkStatus();
absl::Status name_status = absl::UnknownError("");
int name_index = 0;
int name_index = -1;
std::vector<std::string> v = absl::StrSplit(tag_and_name, ':');
if (v.size() == 1) {
name_status = ValidateName(v[0]);
@ -143,7 +143,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
tag_status = ValidateTag(v[0]);
name_status = ValidateName(v[1]);
name_index = 1;
}
} // else omitted, name_index == -1, triggering error.
if (name_index == -1 || tag_status != absl::OkStatus() ||
name_status != absl::OkStatus()) {
tag->clear();

38
mediapipe/gpu/BUILD
View File

@ -511,11 +511,19 @@ cc_library(
],
}),
deps = [
":gl_base_hdr",
":gl_context",
":gl_texture_buffer",
":gl_texture_view",
":gpu_buffer_format",
":gpu_buffer_storage",
":image_frame_view",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:ahwb_view",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//third_party/GL:EGL_headers",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings:str_format",
],
)
@ -526,12 +534,14 @@ mediapipe_proto_library(
visibility = ["//visibility:public"],
)
objc_library(
cc_library(
name = "pixel_buffer_pool_util",
srcs = ["pixel_buffer_pool_util.mm"],
srcs = ["pixel_buffer_pool_util.cc"],
hdrs = ["pixel_buffer_pool_util.h"],
copts = [
"-x objective-c++",
"-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
],
visibility = ["//visibility:public"],
deps = [
@ -542,13 +552,14 @@ objc_library(
],
)
objc_library(
cc_library(
name = "metal_shared_resources",
srcs = ["metal_shared_resources.mm"],
srcs = ["metal_shared_resources.cc"],
hdrs = ["metal_shared_resources.h"],
copts = [
"-x objective-c++",
"-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
],
features = ["-layering_check"],
visibility = ["//visibility:public"],
@ -557,15 +568,17 @@ objc_library(
"@google_toolbox_for_mac//:GTM_Defines",
] + [
],
alwayslink = 1,
)
objc_library(
cc_library(
name = "MPPMetalUtil",
srcs = ["MPPMetalUtil.mm"],
srcs = ["MPPMetalUtil.cc"],
hdrs = ["MPPMetalUtil.h"],
copts = [
"-x objective-c++",
"-Wno-shorten-64-to-32",
"-fobjc-arc", # enable reference-counting
],
visibility = ["//visibility:public"],
deps = [
@ -575,6 +588,7 @@ objc_library(
"@com_google_absl//absl/time",
"@google_toolbox_for_mac//:GTM_Defines",
],
alwayslink = 1,
)
mediapipe_proto_library(
@ -655,6 +669,8 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/status",
] + select({
"//conditions:default": [],
"//mediapipe:apple": [
@ -857,12 +873,14 @@ cc_library(
}),
)
objc_library(
cc_library(
name = "MPPMetalHelper",
srcs = ["MPPMetalHelper.mm"],
srcs = ["MPPMetalHelper.cc"],
hdrs = ["MPPMetalHelper.h"],
copts = [
"-Wno-shorten-64-to-32",
"-x objective-c++",
"-fobjc-arc",
],
features = ["-layering_check"],
visibility = ["//visibility:public"],
@ -1215,9 +1233,13 @@ mediapipe_cc_test(
],
requires_full_emulation = True,
deps = [
":gl_texture_buffer",
":gl_texture_util",
":gpu_buffer_format",
":gpu_buffer_storage_ahwb",
":gpu_test_base",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/tool:test_util",
],
)

60
mediapipe/gpu/MPPMetalHelper.mm → mediapipe/gpu/MPPMetalHelper.cc
View File

@ -14,15 +14,14 @@
#import "mediapipe/gpu/MPPMetalHelper.h"
#import "GTMDefines.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/framework/port/ret_check.h"
#import "mediapipe/gpu/gpu_buffer.h"
#import "mediapipe/gpu/gpu_service.h"
#import "mediapipe/gpu/graph_support.h"
#import "mediapipe/gpu/metal_shared_resources.h"
#import "GTMDefines.h"
#include "mediapipe/framework/port/ret_check.h"
@interface MPPMetalHelper () {
mediapipe::GpuResources* _gpuResources;
@ -31,7 +30,8 @@
namespace mediapipe {
// Using a C++ class so it can be declared as a friend of LegacyCalculatorSupport.
// Using a C++ class so it can be declared as a friend of
// LegacyCalculatorSupport.
class MetalHelperLegacySupport {
public:
static CalculatorContract* GetCalculatorContract() {
@ -61,7 +61,8 @@ class MetalHelperLegacySupport {
- (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc {
if (!cc) return nil;
return [self initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
return [self
initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
}
+ (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc {
@ -77,7 +78,8 @@ class MetalHelperLegacySupport {
}
// Legacy support.
- (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets {
- (instancetype)initWithSidePackets:
(const mediapipe::PacketSet&)inputSidePackets {
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext();
if (cc) {
ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets());
@ -85,16 +87,19 @@ class MetalHelperLegacySupport {
}
// TODO: remove when we can.
ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses "
ABSL_LOG(WARNING)
<< "CalculatorContext not available. If this calculator uses "
"CalculatorBase, call initWithCalculatorContext instead.";
mediapipe::GpuSharedData* gpu_shared =
inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get<mediapipe::GpuSharedData*>();
inputSidePackets.Tag(mediapipe::kGpuSharedTagName)
.Get<mediapipe::GpuSharedData*>();
return [self initWithGpuResources:gpu_shared->gpu_resources.get()];
}
// Legacy support.
+ (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets {
+ (absl::Status)setupInputSidePackets:
(mediapipe::PacketTypeSet*)inputSidePackets {
auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract();
if (cc) {
ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets());
@ -102,11 +107,11 @@ class MetalHelperLegacySupport {
}
// TODO: remove when we can.
ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this "
ABSL_LOG(WARNING)
<< "CalculatorContract not available. If you're calling this "
"from a GetContract method, call updateContract instead.";
auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0);
RET_CHECK(id.IsValid())
<< "A " << mediapipe::kGpuSharedTagName
RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName
<< " input side packet is required here.";
inputSidePackets->Get(id).Set<mediapipe::GpuSharedData*>();
return absl::OkStatus();
@ -125,10 +130,12 @@ class MetalHelperLegacySupport {
}
- (id<MTLCommandBuffer>)commandBuffer {
return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
return
[_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
}
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer
plane:(size_t)plane {
CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer);
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
@ -178,40 +185,47 @@ class MetalHelperLegacySupport {
CVMetalTextureRef texture;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
NULL, _gpuResources->metal_shared().resources().mtlTextureCache,
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane,
&texture);
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width,
height, plane, &texture);
ABSL_CHECK_EQ(err, kCVReturnSuccess);
return texture;
}
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer {
return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0];
}
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
- (id<MTLTexture>)metalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer {
return [self metalTextureWithGpuBuffer:gpuBuffer plane:0];
}
- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
- (id<MTLTexture>)metalTextureWithGpuBuffer:
(const mediapipe::GpuBuffer&)gpuBuffer
plane:(size_t)plane {
CFHolder<CVMetalTextureRef> cvTexture;
cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]);
return CVMetalTextureGetTexture(*cvTexture);
}
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height {
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
height:(int)height {
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height);
}
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
height:(int)height
format:(mediapipe::GpuBufferFormat)format {
format:(mediapipe::GpuBufferFormat)
format {
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format);
}
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error {
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name
error:(NSError* _Nullable*)error {
return [_gpuResources->metal_shared().resources().mtlDevice
newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name
newLibraryWithFile:[[NSBundle bundleForClass:[self class]]
pathForResource:name
ofType:@"metallib"]
error:error];
}

8
mediapipe/gpu/MPPMetalUtil.mm → mediapipe/gpu/MPPMetalUtil.cc
View File

@ -69,10 +69,10 @@
while (!bufferCompleted) {
auto duration = absl::Now() - start_time;
// If the spin-lock takes more than 5 ms then go to blocking wait:
// - it frees the CPU core for another threads: increase the performance/decrease power
// consumption.
// - if a driver thread that notifies that the GPU buffer is completed has lower priority then
// the CPU core is allocated for the thread.
// - it frees the CPU core for another threads: increase the
// performance/decrease power consumption.
// - if a driver thread that notifies that the GPU buffer is completed has
// lower priority then the CPU core is allocated for the thread.
if (duration >= absl::Milliseconds(5)) {
[commandBuffer waitUntilCompleted];
break;

4
mediapipe/gpu/gl_calculator_helper.cc
View File

@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
// static
absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
bool requesst_gpu_as_optional) {
if (requesst_gpu_as_optional) {
bool request_gpu_as_optional) {
if (request_gpu_as_optional) {
cc->UseService(kGpuService).Optional();
} else {
cc->UseService(kGpuService);

2
mediapipe/gpu/gl_calculator_helper.h
View File

@ -68,7 +68,7 @@ class GlCalculatorHelper {
// This method can be called from GetContract to set up the needed GPU
// resources.
static absl::Status UpdateContract(CalculatorContract* cc,
bool requesst_gpu_as_optional = false);
bool request_gpu_as_optional = false);
// This method can be called from FillExpectations to set the correct types
// for the shared GL input side packet(s).

9
mediapipe/gpu/gl_texture_buffer.cc
View File

@ -14,6 +14,8 @@
#include "mediapipe/gpu/gl_texture_buffer.h"
#include <cstdint>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h"
@ -131,6 +133,13 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
SymbolAvailable(&glTexStorage2D)) {
ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else if (info.immutable) {
ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
context->GetGlVersion() != GlVersion::kGLES2)
<< "Immutable GpuBuffer format requested is not supported in this "
<< "GlContext. Format was " << static_cast<uint32_t>(format_);
ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else {
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
height_, 0 /* border */, info.gl_format, info.gl_type, data);

1
mediapipe/gpu/gl_texture_buffer.h
View File

@ -19,6 +19,7 @@
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
#include <atomic>
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"

2
mediapipe/gpu/gl_thread_collector.h
View File

@ -17,7 +17,7 @@
#include <cstdlib>
#if defined(MEDIAPIPE_USING_SWIFTSHADER)
#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
#endif

16
mediapipe/gpu/gpu_buffer_format.cc
View File

@ -35,6 +35,10 @@ namespace mediapipe {
#endif // GL_HALF_FLOAT_OES
#endif // __EMSCRIPTEN__
#ifndef GL_RGBA8
#define GL_RGBA8 0x8058
#endif // GL_RGBA8
#if !MEDIAPIPE_DISABLE_GPU
#ifdef GL_ES_VERSION_2_0
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
@ -163,6 +167,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
{
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
}},
{GpuBufferFormat::kImmutableRGBAFloat128,
{
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
}},
{GpuBufferFormat::kImmutableRGBA32,
{
{GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
}},
}};
static const auto* gles2_format_info = ([] {
@ -206,6 +218,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
case GpuBufferFormat::kImmutableRGBA32:
case GpuBufferFormat::kBGRA32:
// TODO: verify we are handling order of channels correctly.
return ImageFormat::SRGBA;
@ -221,10 +234,11 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
return ImageFormat::SRGB;
case GpuBufferFormat::kTwoComponentFloat32:
return ImageFormat::VEC32F2;
case GpuBufferFormat::kImmutableRGBAFloat128:
case GpuBufferFormat::kRGBAFloat128:
return ImageFormat::VEC32F4;
case GpuBufferFormat::kRGBA32:
// TODO: this likely maps to ImageFormat::SRGBA
return ImageFormat::SRGBA;
case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kOneComponent8Alpha:
case GpuBufferFormat::kOneComponent8Red:

9
mediapipe/gpu/gpu_buffer_format.h
View File

@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t {
kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible.
kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'),
kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
// Immutable version of kRGBA32
kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'),
// Immutable version of kRGBAFloat128
kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'),
// 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling.
kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'),
// 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling.
@ -78,6 +82,9 @@ struct GlTextureInfo {
// For multiplane buffers, this represents how many times smaller than
// the nominal image size a plane is.
int downscale;
// For GLES3.1+ compute shaders, users may explicitly request immutable
// textures.
bool immutable = false;
};
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
return kCVPixelFormatType_64RGBAHalf;
case GpuBufferFormat::kRGBAFloat128:
return kCVPixelFormatType_128RGBAFloat;
case GpuBufferFormat::kImmutableRGBA32:
case GpuBufferFormat::kImmutableRGBAFloat128:
case GpuBufferFormat::kNV12:
case GpuBufferFormat::kNV21:
case GpuBufferFormat::kI420:

2
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
View File

@ -151,7 +151,7 @@ static std::shared_ptr<GpuBufferStorageCvPixelBuffer> ConvertFromImageFrame(
std::shared_ptr<GpuBufferStorageImageFrame> frame) {
auto status_or_buffer =
CreateCVPixelBufferForImageFrame(frame->image_frame());
ABSL_CHECK(status_or_buffer.ok());
ABSL_CHECK_OK(status_or_buffer);
return std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(status_or_buffer).value());
}

3
mediapipe/gpu/gpu_origin.proto
View File

@ -16,6 +16,9 @@ syntax = "proto3";
package mediapipe;
option java_package = "com.google.mediapipe.gpu";
option java_outer_classname = "GpuOriginProto";
message GpuOrigin {
enum Mode {
DEFAULT = 0;

55
mediapipe/gpu/gpu_shared_data_internal.cc
View File

@ -14,10 +14,14 @@
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include <memory>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/status/status.h"
#include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_options.pb.h"
#include "mediapipe/gpu/graph_support.h"
@ -83,8 +87,25 @@ GpuResources::StatusOrGpuResources GpuResources::Create(
}
GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
: gl_key_context_(new GlContextMapType(),
[](auto* map) {
// This flushes all pending jobs in all GL contexts,
// ensuring that all GL contexts not referenced
// elsewhere are destroyed as part of this destructor.
// Failure to do this may cause GL threads to outlast
// this destructor and execute jobs after the
// GpuResources object is destroyed.
for (auto& [key, context] : *map) {
const auto status = std::move(context)->Run(
[]() { return absl::OkStatus(); });
ABSL_LOG_IF(ERROR, !status.ok())
<< "Failed to flush GlContext jobs: " << status;
}
delete map;
})
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
: texture_caches_(std::make_shared<CvTextureCacheManager>()),
,
texture_caches_(std::make_shared<CvTextureCacheManager>()),
gpu_buffer_pool_(
[tc = texture_caches_](const internal::GpuBufferSpec& spec,
const MultiPoolOptions& options) {
@ -92,7 +113,7 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
})
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
{
gl_key_context_[SharedContextKey()] = gl_context;
gl_key_context_->insert({SharedContextKey(), gl_context});
named_executors_[kGpuExecutorName] =
std::make_shared<GlContextExecutor>(gl_context.get());
#if __APPLE__
@ -104,6 +125,15 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
}
GpuResources::~GpuResources() {
// This flushes all pending jobs in all GL contexts,
// ensuring that all existing jobs, which may refer GpuResource and kept their
// gpu resources (e.g. GpuResources::gpu_buffer_pool_) through a raw pointer,
// have finished before kept gpu resources get deleted.
for (auto& [key, context] : *gl_key_context_) {
const auto status = context->Run([]() { return absl::OkStatus(); });
ABSL_LOG_IF(ERROR, !status.ok())
<< "Failed to flush GlContext jobs: " << status;
}
#if __APPLE__
// Note: on Apple platforms, this object contains Objective-C objects.
// The destructor will release them, but ARC must be on.
@ -111,7 +141,7 @@ GpuResources::~GpuResources() {
#error This file must be built with ARC.
#endif
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
for (auto& kv : gl_key_context_) {
for (auto& kv : *gl_key_context_) {
texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache());
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@ -173,23 +203,24 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
const std::shared_ptr<GlContext>& GpuResources::gl_context(
CalculatorContext* cc) {
if (cc) {
auto it = gl_key_context_.find(node_key_[cc->NodeName()]);
if (it != gl_key_context_.end()) {
auto it = gl_key_context_->find(node_key_[cc->NodeName()]);
if (it != gl_key_context_->end()) {
return it->second;
}
}
return gl_key_context_[SharedContextKey()];
return gl_key_context_->at(SharedContextKey());
}
GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext(
const std::string& key) {
auto it = gl_key_context_.find(key);
if (it == gl_key_context_.end()) {
MP_ASSIGN_OR_RETURN(std::shared_ptr<GlContext> new_context,
GlContext::Create(*gl_key_context_[SharedContextKey()],
auto it = gl_key_context_->find(key);
if (it == gl_key_context_->end()) {
MP_ASSIGN_OR_RETURN(
std::shared_ptr<GlContext> new_context,
GlContext::Create(*gl_key_context_->at(SharedContextKey()),
kGlContextUseDedicatedThread));
it = gl_key_context_.emplace(key, new_context).first;
it = gl_key_context_->emplace(key, new_context).first;
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
texture_caches_->RegisterTextureCache(it->second->cv_texture_cache());
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER

7
mediapipe/gpu/gpu_shared_data_internal.h
View File

@ -21,6 +21,8 @@
#ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#include <memory>
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_node.h"
#include "mediapipe/framework/executor.h"
@ -82,7 +84,10 @@ class GpuResources {
const std::string& ContextKey(const std::string& canonical_node_name);
std::map<std::string, std::string> node_key_;
std::map<std::string, std::shared_ptr<GlContext>> gl_key_context_;
using GlContextMapType = std::map<std::string, std::shared_ptr<GlContext>>;
std::unique_ptr<GlContextMapType, void (*)(GlContextMapType*)>
gl_key_context_;
#ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
std::shared_ptr<CvTextureCacheManager> texture_caches_;

13
mediapipe/gpu/gpu_test_base.h
View File

@ -15,6 +15,9 @@
#ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_
#define MEDIAPIPE_GPU_GPU_TEST_BASE_H_
#include <functional>
#include <memory>
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
@ -22,9 +25,9 @@
namespace mediapipe {
class GpuTestBase : public ::testing::Test {
class GpuTestEnvironment {
protected:
GpuTestBase() { helper_.InitializeForTest(gpu_resources_.get()); }
GpuTestEnvironment() { helper_.InitializeForTest(gpu_resources_.get()); }
void RunInGlContext(std::function<void(void)> gl_func) {
helper_.RunInGlContext(std::move(gl_func));
@ -35,6 +38,12 @@ class GpuTestBase : public ::testing::Test {
GlCalculatorHelper helper_;
};
class GpuTestBase : public testing::Test, public GpuTestEnvironment {};
template <typename T>
class GpuTestWithParamBase : public testing::TestWithParam<T>,
public GpuTestEnvironment {};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_TEST_BASE_H_

2
mediapipe/gpu/metal.bzl
View File

@ -171,7 +171,7 @@ METAL_LIBRARY_ATTRS = dicts.add(apple_support.action_required_attrs(), {
metal_library = rule(
implementation = _metal_library_impl,
attrs = METAL_LIBRARY_ATTRS,
fragments = ["apple", "objc", "swift"],
fragments = ["apple", "objc"],
output_to_genfiles = True,
)
"""

7
mediapipe/gpu/metal_shared_resources.mm → mediapipe/gpu/metal_shared_resources.cc
View File

@ -50,9 +50,10 @@
- (CVMetalTextureCacheRef)mtlTextureCache {
@synchronized(self) {
if (!_mtlTextureCache) {
CVReturn __unused err =
CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d ; device %@", err,
CVReturn __unused err = CVMetalTextureCacheCreate(
NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
NSAssert(err == kCVReturnSuccess,
@"Error at CVMetalTextureCacheCreate %d ; device %@", err,
self.mtlDevice);
// TODO: register and flush metal caches too.
}

58
mediapipe/gpu/pixel_buffer_pool_util.mm → mediapipe/gpu/pixel_buffer_pool_util.cc
View File

@ -24,23 +24,27 @@
namespace mediapipe {
CVPixelBufferPoolRef CreateCVPixelBufferPool(
int width, int height, OSType pixelFormat, int keepCount,
CVPixelBufferPoolRef CreateCVPixelBufferPool(int width, int height,
OSType pixelFormat, int keepCount,
CFTimeInterval maxAge) {
CVPixelBufferPoolRef pool = NULL;
NSMutableDictionary *sourcePixelBufferOptions =
[(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy];
[(__bridge NSDictionary *)GetCVPixelBufferAttributesForGlCompatibility()
mutableCopy];
[sourcePixelBufferOptions addEntriesFromDictionary:@{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height),
}];
NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init];
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount);
NSMutableDictionary *pixelBufferPoolOptions =
[[NSMutableDictionary alloc] init];
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] =
@(keepCount);
if (maxAge > 0) {
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = @(maxAge);
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] =
@(maxAge);
}
CVPixelBufferPoolCreate(
@ -50,8 +54,9 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
return pool;
}
OSStatus PreallocateCVPixelBufferPoolBuffers(
CVPixelBufferPoolRef pool, int count, CFDictionaryRef auxAttributes) {
OSStatus PreallocateCVPixelBufferPoolBuffers(CVPixelBufferPoolRef pool,
int count,
CFDictionaryRef auxAttributes) {
CVReturn err = kCVReturnSuccess;
NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init];
for (int i = 0; i < count && err == kCVReturnSuccess; i++) {
@ -68,30 +73,37 @@ OSStatus PreallocateCVPixelBufferPoolBuffers(
return err;
}
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(int allocationThreshold) {
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(
int allocationThreshold) {
if (allocationThreshold > 0) {
return (CFDictionaryRef)CFBridgingRetain(
@{(id)kCVPixelBufferPoolAllocationThresholdKey: @(allocationThreshold)});
return (CFDictionaryRef)CFBridgingRetain(@{
(id)kCVPixelBufferPoolAllocationThresholdKey : @(allocationThreshold)
});
} else {
return nil;
}
}
CVReturn CreateCVPixelBufferWithPool(
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
CVTextureCacheType textureCache, CVPixelBufferRef* outBuffer) {
return CreateCVPixelBufferWithPool(pool, auxAttributes, [textureCache](){
CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
CFDictionaryRef auxAttributes,
CVTextureCacheType textureCache,
CVPixelBufferRef *outBuffer) {
return CreateCVPixelBufferWithPool(
pool, auxAttributes,
[textureCache]() {
#if TARGET_OS_OSX
CVOpenGLTextureCacheFlush(textureCache, 0);
#else
CVOpenGLESTextureCacheFlush(textureCache, 0);
#endif // TARGET_OS_OSX
}, outBuffer);
},
outBuffer);
}
CVReturn CreateCVPixelBufferWithPool(
CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
std::function<void(void)> flush, CVPixelBufferRef* outBuffer) {
CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
CFDictionaryRef auxAttributes,
std::function<void(void)> flush,
CVPixelBufferRef *outBuffer) {
CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(
kCFAllocatorDefault, pool, auxAttributes, outBuffer);
if (err == kCVReturnWouldExceedAllocationThreshold) {
@ -103,10 +115,12 @@ CVReturn CreateCVPixelBufferWithPool(
kCFAllocatorDefault, pool, auxAttributes, outBuffer);
}
if (err == kCVReturnWouldExceedAllocationThreshold) {
// TODO: allow the application to set the threshold. For now, disable it by
// default, since the threshold we are using is arbitrary and some graphs routinely cross it.
// TODO: allow the application to set the threshold. For now, disable it
// by default, since the threshold we are using is arbitrary and some
// graphs routinely cross it.
#ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK
NSLog(@"Using more buffers than expected! This is a debug-only warning, "
NSLog(
@"Using more buffers than expected! This is a debug-only warning, "
"you can ignore it if your app works fine otherwise.");
#ifdef DEBUG
NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description);

92
mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
View File

@ -17,7 +17,6 @@ package com.google.mediapipe.components;
import static java.lang.Math.max;
import static java.lang.Math.min;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
@ -25,6 +24,7 @@ import android.opengl.GLES31;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import android.util.Pair;
import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.glutil.CommonShaders;
import com.google.mediapipe.glutil.ShaderUtil;
@ -51,11 +51,9 @@ import javax.microedition.khronos.opengles.GL10;
* {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}).
*/
public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
/**
* Listener for Bitmap capture requests.
*/
public interface BitmapCaptureListener {
void onBitmapCaptured(Bitmap result);
/** Listener for image capture requests. */
public interface ImageCaptureListener {
void onImageCaptured(int width, int height, int[] data);
}
/**
@ -86,27 +84,28 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
private float[] textureTransformMatrix = new float[16];
private SurfaceTexture surfaceTexture = null;
private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean();
private BitmapCaptureListener bitmapCaptureListener;
private final AtomicBoolean captureNextFrameImage = new AtomicBoolean();
private ImageCaptureListener imageCaptureListener;
// Specifies whether a black CLAMP_TO_BORDER effect should be used.
private boolean shouldClampToBorder = false;
private Scale scale = Scale.FILL;
/**
* Sets the {@link BitmapCaptureListener}.
*/
public void setBitmapCaptureListener(BitmapCaptureListener bitmapCaptureListener) {
this.bitmapCaptureListener = bitmapCaptureListener;
private float zoomFactor = 1.0f;
private Pair<Float, Float> zoomLocation = new Pair<>(0.5f, 0.5f);
/** Sets the {@link ImageCaptureListener}. */
public void setImageCaptureListener(ImageCaptureListener imageCaptureListener) {
this.imageCaptureListener = imageCaptureListener;
}
/**
* Request to capture Bitmap of the next frame.
* Request to capture Image of the next frame.
*
* The result will be provided to the {@link BitmapCaptureListener} if one is set. Please note
* <p>The result will be provided to the {@link ImageCaptureListener} if one is set. Please note
* this is an expensive operation and the result may not be available for a while.
*/
public void captureNextFrameBitmap() {
captureNextFrameBitmap.set(true);
public void captureNextFrameImage() {
captureNextFrameImage.set(true);
}
@Override
@ -202,9 +201,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
ShaderUtil.checkGlError("glDrawArrays");
// Capture Bitmap if requested.
BitmapCaptureListener bitmapCaptureListener = this.bitmapCaptureListener;
if (captureNextFrameBitmap.getAndSet(false) && bitmapCaptureListener != null) {
// Capture image if requested.
ImageCaptureListener imageCaptureListener = this.imageCaptureListener;
if (captureNextFrameImage.getAndSet(false) && imageCaptureListener != null) {
// Find the name of the bound texture.
int[] texName = new int[1];
if (surfaceTexture != null) {
@ -219,8 +218,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1);
int texWidth = texDims[0];
int texHeight = texDims[1];
int bitmapSize = texWidth * texHeight;
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(bitmapSize * 4);
int imageSize = texWidth * texHeight;
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(imageSize * 4);
byteBuffer.order(ByteOrder.nativeOrder());
// Read pixels from texture.
@ -235,27 +234,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
GLES20.glDeleteFramebuffers(1, fbo, 0);
ShaderUtil.checkGlError("capture frame");
int[] pixelBuffer = new int[bitmapSize];
byteBuffer.asIntBuffer().get(pixelBuffer);
for (int i = 0; i < bitmapSize; i++) {
// Swap R and B channels.
pixelBuffer[i] =
(pixelBuffer[i] & 0xff00ff00)
| ((pixelBuffer[i] & 0x000000ff) << 16)
| ((pixelBuffer[i] & 0x00ff0000) >> 16);
}
// Send bitmap.
Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888);
bitmap.setPixels(
pixelBuffer,
/* offset= */ bitmapSize - texWidth,
/* stride= */ -texWidth,
/* x= */ 0,
/* y= */ 0,
texWidth,
texHeight);
bitmapCaptureListener.onBitmapCaptured(bitmap);
int[] data = new int[imageSize];
byteBuffer.asIntBuffer().get(data);
imageCaptureListener.onImageCaptured(texWidth, texHeight, data);
}
GLES20.glBindTexture(textureTarget, 0);
@ -294,6 +275,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
float textureBottom = (1.0f - scaleHeight) * alignmentVertical;
float textureTop = textureBottom + scaleHeight;
Pair<Float, Float> currentZoomLocation = this.zoomLocation;
float zoomLocationX = currentZoomLocation.first;
float zoomLocationY = currentZoomLocation.second;
textureLeft = (textureLeft - 0.5f) / zoomFactor + zoomLocationX;
textureRight = (textureRight - 0.5f) / zoomFactor + zoomLocationX;
textureBottom = (textureBottom - 0.5f) / zoomFactor + zoomLocationY;
textureTop = (textureTop - 0.5f) / zoomFactor + zoomLocationY;
return new float[] {textureLeft, textureRight, textureBottom, textureTop};
}
@ -380,6 +370,22 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
this.scale = scale;
}
/** Zoom factor applied to the frame, must not be 0. */
public void setZoomFactor(float zoomFactor) {
if (zoomFactor == 0.f) {
return;
}
this.zoomFactor = zoomFactor;
}
/**
* Location where to apply the zooming of the frame to. Default is 0.5, 0.5 (scaling is applied to
* the center).
*/
public void setZoomLocation(float zoomLocationX, float zoomLocationY) {
this.zoomLocation = new Pair<>(zoomLocationX, zoomLocationY);
}
private boolean isExternalTexture() {
return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
}

10
mediapipe/java/com/google/mediapipe/framework/PacketGetter.java
View File

@ -128,6 +128,16 @@ public final class PacketGetter {
return ProtoUtil.unpack(result, defaultInstance);
}
public static <T extends MessageLite> T getProto(final Packet packet, Parser<T> messageParser) {
SerializedMessage result = new SerializedMessage();
nativeGetProto(packet.getNativeHandle(), result);
try {
return messageParser.parseFrom(result.value);
} catch (InvalidProtocolBufferException e) {
throw new IllegalArgumentException(e);
}
}
/**
* @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds.
*/

Some files were not shown because too many files have changed in this diff Show More